[ Feb 28 13:17:01 Enabled. ] [ Feb 28 13:17:01 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Feb 28 13:17:01 Method "start" exited with status 0. ] [ Feb 28 13:17:01 Rereading configuration. ] [ Feb 28 13:17:01 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:01.956056856Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:247"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.009473164Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:254","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.009497654Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.120181388Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.128773339Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.137123954Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:264","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.739836282Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.880035317Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.880088028Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.88017218Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.880266953Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.880293523Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.880336035Z","hostname":"buskin","pid":1671,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.880358945Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883065145Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:02.883084195Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883090346Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883095096Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883110436Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883122716Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883131657Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883137307Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883144707Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883151207Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883160867Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883165397Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883169828Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883174158Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883179888Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883185068Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883189968Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"None"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:02.883198578Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.062897071Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.062951892Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.062960312Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238492938Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238538659Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238550199Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238555139Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.23856109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238565679Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.23857645Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.23858127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.23859295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.2385986Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238636592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238654032Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238735014Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 489984, free: 20937475584, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 935424, free: 20937030144, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238770325Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238855027Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238885798Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238894828Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238913928Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238928909Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.238933919Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.2389719Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.23898216Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.253241858Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:700"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.253272989Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999951139s","next_cleanup":"Instant { tv_sec: 17518, tv_nsec: 124155465 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.25332168Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3407"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.253378621Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.255075785Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.255144327Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256091931Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256193494Z","hostname":"buskin","pid":1671,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256450421Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256459481Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256530383Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256586064Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256608455Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256615945Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256638076Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256652226Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256673697Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256687787Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256691067Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256694527Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256716347Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.256739378Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.257438656Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.25758731Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.25759583Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.257630791Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.257639181Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.257789235Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.257912898Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.25796191Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 2c331470-b440-4c33-9aa1-35acb298a49d, rack_id: 9ebadecf-c131-4403-83ea-910555047c33, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.258064282Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:45825","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 2c331470-b440-4c33-9aa1-35acb298a49d, rack_id: 9ebadecf-c131-4403-83ea-910555047c33, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.258217306Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.258223056Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.258262867Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/server.rs:47"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.258272338Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.258275718Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.258279968Z","hostname":"buskin","pid":1671,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.260906875Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3635"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.266924501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.267012443Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.267019123Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.267046554Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.267147487Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.267153256Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.297967761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.3056791Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.381844343Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:342"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.381886165Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.410311207Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.432604042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.432668094Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.432695574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.432730215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.432859959Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.432869309Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.460249815Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.467780729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.569483741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.591844057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.591894308Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.591910419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.591929959Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.592085473Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.592103014Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.619000847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.626156162Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.729680031Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.751099193Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.751134324Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.751153464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.751166345Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.751299898Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.751307998Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.777486203Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.785096739Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.88596911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.90810163Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.908202083Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.908216713Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.908244384Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.908355837Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.908366697Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.935940898Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:03.94339362Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.045763019Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.067895979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.067963571Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.067973461Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.068000662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.068137316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.068144326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.094622519Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.101753212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.201450982Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.223021659Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.223108091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.223120511Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.223135372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.223251555Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.223258995Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.249729207Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.256974794Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.359331673Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.380702694Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.380760405Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.380770165Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.380782196Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.380908149Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.380914919Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.408316386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.415602214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.516778232Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.538114342Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.538154973Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.538167563Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.538182894Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.538317077Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.538332298Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.565537169Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.572808566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.675588066Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.69009756Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.690145411Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.690156952Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.692844711Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 489984, free: 20937475584, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 935424, free: 20937030144, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 771072, free: 20937194496, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 1210368, free: 20936755200, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 775680, free: 20937189888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 780288, free: 20937185280, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.712777795Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.715527815Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.732721059Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.736395164Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.736438735Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.741164247Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:369"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.741177407Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.741182887Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:04.804418727Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.02370217Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.037743972Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.037827694Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.037854355Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.037870445Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.205287252Z","hostname":"buskin","pid":1671,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.743905817Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.744438461Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.744483432Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:206"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.744492982Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:214"} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.757952819Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.759402277Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.759737425Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.759946541Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.760098975Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.76028684Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.760475174Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.760720231Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.760826253Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.760930026Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761044049Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761147262Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761247494Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761368107Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.7614667Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761565622Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761674915Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761782618Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.761889741Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.762010454Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.762121657Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:05.762214159Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:08.606417053Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 30.708949ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:08.882427869Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 141.007666ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:09.091679603Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 236.663093ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:09.28006796Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 218.475664ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:09.728731546Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 646.601386ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:09.991227553Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 949.871302ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:10.711220865Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.242464514s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:11.841850662Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.003977292s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:13.212398275Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:15.460635624Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1949"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.154524233Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:981","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.154628796Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211317847Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211363118Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211372678Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211387209Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.21142163Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211695337Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211883692Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:137"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211910612Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211918003Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3407"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.211930173Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:45825","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.215049033Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.215090374Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.215277989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.215353441Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.215386592Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.215405602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:16.215621468Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"286.556545ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.216637444Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:735","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.247206962Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44821"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.247469229Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.247557081Z","hostname":"buskin","pid":1671,"uri":"/zpools","method":"GET","req_id":"7e736844-d334-403f-8add-015cda37bf3f","remote_addr":"[fd00:1122:3344:101::1]:44821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.276224261Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.276348724Z","hostname":"buskin","pid":1671,"uri":"/sled-role","method":"GET","req_id":"8ab2d74c-7620-4e18-abd9-5d1911518c26","remote_addr":"[fd00:1122:3344:101::1]:43710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":79,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.278498139Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.278509949Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.278793037Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.279046433Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:741"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.282069591Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.310164596Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.3103404Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47872"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.310637318Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.310654988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.356843989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.35688389Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.358912842Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.399928539Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.399973821Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.442841686Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.442882187Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.442889287Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.442949979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.480418605Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.481854322Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.481861642Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.481870652Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:16.50389987Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"345.445342ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.535672489Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.539721174Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.543681396Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.556495586Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.561132486Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.565284613Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.7226598Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:16.85223842Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"781.782097ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:16.935153018Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:17.160655771Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:17.579999222Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:17.635086842Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"1.359560177s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:17.859663772Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:18.302037866Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:18.396237824Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:18.401070719Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:18.417991075Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 33.433502ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.448143852Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 78.710823ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.491246353Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 50.127108ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.542133545Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 71.980969ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.56325969Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 272.954405ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.591083757Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 68.682492ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.616058471Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 60.129728ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.807125847Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 171.388817ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.816341994Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 172.245413ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.842426707Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 497.309316ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:18.889893701Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 401.721936ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.008365745Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.012022079Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"3.924154805s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 417.229527ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.051836156Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:19.05199333Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 495.26126ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.396432069Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 552.436301ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.417740899Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 741.73656ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.476799461Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:19.68580862Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 737.850985ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.901992733Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:19.978054194Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1966"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:19.978103335Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 746.701671ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:19.980459856Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:20.110635401Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 970.712545ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:20.2304283Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 971.462665ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:20.649811112Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 1.406325171s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:20.735959222Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:20.972178242Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 585.890303ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:21.209368787Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 1.281419251s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:21.629947429Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 697.578148ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:21.802754424Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 573.063388ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:22.150570841Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:22.326371013Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"wait for service Some(\"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9\") failed: Property not found. retry in 862.829895ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:22.510374437Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150\") failed: Property not found. retry in 1.21200813s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:22.73390266Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080\") failed: Property not found. retry in 1.014591487s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:22.921330532Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:22.937547979Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"5.139136846s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:23.419448553Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:23.419497264Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:23.985326411Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:23.985375182Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.181340564Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe43:ef6","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.508123349Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1973"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.513757004Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.635740929Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2466"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.709969382Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.710348562Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.760014313Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:2::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.760071334Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:24.900068743Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2345"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.188956141Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2380"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.307548238Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.307584239Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.38092603Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2647"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.481866652Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.524104271Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2652"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.703420374Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2586"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.867859523Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2374"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.877119042Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3659","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.877178153Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3675"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:25.877197904Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.034380796Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:3::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.034437067Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.423124817Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3683"} {"msg":"configuring dendrite zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.428611949Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3739"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.582120496Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.582164198Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.607164672Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3779","rack_subnet":"fd00:1122:3344::"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.656264208Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 63.112316ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:26.68814084Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe6114a0-5b69-4d87-a81f-11254b1e2cc4\", \"content-length\": \"152\", \"date\": \"Wed, 28 Feb 2024 13:17:26 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"fe6114a0-5b69-4d87-a81f-11254b1e2cc4\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 67.348ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:26.752418037Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c1c3cfb-1dff-4f11-b35b-60f147c3f38c\", \"content-length\": \"152\", \"date\": \"Wed, 28 Feb 2024 13:17:26 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"2c1c3cfb-1dff-4f11-b35b-60f147c3f38c\" }"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:26.756558693Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Failed to get switch ID from MGS (retrying in 262.347319ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:26.821444036Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a0473dc-f293-458c-94bf-53be6be77b02\", \"content-length\": \"152\", \"date\": \"Wed, 28 Feb 2024 13:17:26 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"9a0473dc-f293-458c-94bf-53be6be77b02\" }"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.085531754Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.117710624Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.117789496Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.118247388Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.118280229Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.195402967Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.231931728Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.279982267Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_1f905b68-a2ec-4d9a-b15d-e713ba321080"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.280036239Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_842e2f1e-ea91-42d4-acc8-64d93d1c76c9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.280043819Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_9e69a966-aaa5-463c-adce-1c59004ff150"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.280115491Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.280224804Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.280424609Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.280755407Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"6fba1391-5fca-450b-abcd-bbe404d39a79","remote_addr":"[fd00:1122:3344:101::1]:47872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":10970335,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.282647136Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.32741594Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.372038291Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.416920568Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.416932968Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.448601194Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.449390995Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.449399285Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.47752817Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.477722485Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45603"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.477864219Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.477885579Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.499337252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.499394714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.517315106Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.517370527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.535249398Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.53530677Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.535516785Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.535632478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.54694193Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.57913719Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.579767776Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.579784336Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.624172071Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 158, 48]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.624229562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.624967101Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.624983892Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.625781172Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.626968553Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.647678327Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:27.665915847Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:28.733095029Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 63.82974ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:28.872673357Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 137.266228ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:28.953321716Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 256.921486ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:29.101204638Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 574.059522ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:29.370998333Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 564.531841ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:29.954200719Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:30.473613528Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39113"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:30.473741461Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"725ea955-6eb7-4dca-9493-6f9987a1ed09","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39113","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":56,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 930.827068ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:30.529344075Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 1.115822053s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:31.46979098Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240\") failed: Property not found. retry in 1.184713713s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:32.596089415Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:33.822447071Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::1b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:33.822520953Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:34.371523766Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:34.373189159Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Set up NTP service boundary=true, Servers=[\"0.pool.ntp.org\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:34.572707643Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2270"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:34.598736474Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.191182377Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.19130626Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.191603488Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.191865954Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"b243c135-35ea-43df-8d40-5afeb89f802b","remote_addr":"[fd00:1122:3344:101::1]:45603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":13714113,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.194145213Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.222963036Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.223131331Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42189"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_01d1c8e5-8905-4e34-824f-8a30fe31e240': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.231714222Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3393"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.231867986Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"a6e1aeb7-c326-40ac-9315-ea8522e14faa","remote_addr":"[fd00:1122:3344:101::1]:42189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8647,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:41.234595636Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.381990546Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.382115559Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48082"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.389274914Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"23e41a76-d3f7-44d6-b96b-986e1d0d2d38","remote_addr":"[fd00:1122:3344:101::1]:48082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7006,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.390776273Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:41.390782053Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.521385689Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.521527133Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51866"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.529828837Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"8cb372fa-3994-4419-8b96-9a173dbae068","remote_addr":"[fd00:1122:3344:101::1]:51866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8219,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.531724646Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:41.531730236Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.638855438Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.638972071Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51369"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.646054983Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"1342a16f-c1bc-40f4-8d6a-fe6c72ffc66f","remote_addr":"[fd00:1122:3344:101::1]:51369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":6998,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.649618995Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:41.649626256Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.808937123Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.809082666Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.817671718Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"af784cc3-0b92-40a5-b9be-f31a4828a3bb","remote_addr":"[fd00:1122:3344:101::1]:45841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8515,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:41.819433453Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:41.819443344Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.053922208Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.05400954Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45587"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.061540715Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"bede6ce0-023d-4768-984a-666def20272b","remote_addr":"[fd00:1122:3344:101::1]:45587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7477,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.063521216Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:42.063528826Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.264533088Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.264651231Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.272149024Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"0203c73c-9f36-4ffc-9dd7-05aae8015653","remote_addr":"[fd00:1122:3344:101::1]:52115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7383,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.274930756Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:42.274938436Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.534027035Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.534128838Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.54275042Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"096497a5-3a68-4783-b9f6-1236de00516c","remote_addr":"[fd00:1122:3344:101::1]:44927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8541,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.545802809Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:42.545809999Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.951988061Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.952098233Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.96009483Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"8de30fe1-0cbb-4a7e-8ab2-4a859e5360e8","remote_addr":"[fd00:1122:3344:101::1]:44836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7841,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:42.963112377Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:42.963125798Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:43.539898907Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:43.539993579Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54623"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:43.547618716Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"f413f6bb-e8ba-4cbd-8667-c28fe055c681","remote_addr":"[fd00:1122:3344:101::1]:54623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7479,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:43.549564276Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:43.549574746Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.197091569Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.197202282Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.206073281Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"09ba7682-830d-446f-a230-92e708282a08","remote_addr":"[fd00:1122:3344:101::1]:58142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8780,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.207882347Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:44.207888758Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.726282512Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.726389605Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54243"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.733937739Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"4ee0f920-4c38-4619-a60f-06426660354d","remote_addr":"[fd00:1122:3344:101::1]:54243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7439,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:44.737582973Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:44.737592084Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:45.802784204Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:45.802883287Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53749"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:45.810976935Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"22a191e7-ebe1-4679-8dc6-e8fc80631e9f","remote_addr":"[fd00:1122:3344:101::1]:53749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7941,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:45.8126985Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:45.81270595Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:47.482436358Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:47.48253406Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35891"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:47.485550808Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:47.490030274Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"c975951e-daa0-497e-86f4-244317217f27","remote_addr":"[fd00:1122:3344:101::1]:35891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7304,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709126259.668671, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:47.491765418Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:47.491775639Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.12373415Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.123911595Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57099"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.128257697Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.144550577Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"4d585741-dfd4-4ffa-996c-c88eed68dcc4","remote_addr":"[fd00:1122:3344:101::1]:57099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":20573,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2680921797, ip_addr: 159.203.158.197, stratum: 3, ref_time: 1709126267.7538416, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.147457182Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.147465792Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.177745403Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.177981438Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34056"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.178352288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.178367299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.198747034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.200684164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.215604928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.21565966Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.231504228Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.23155832Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.28897312Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.289023691Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.341140035Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.341209206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.391915944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.391967135Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.443192636Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.443240747Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.492063796Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.492130267Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.492317782Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.492445856Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.573828394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.573921206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.573931336Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.573938426Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.573949137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.662483819Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.666716508Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.671089351Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.675396422Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.67996123Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.684456835Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.689036124Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.693590171Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.698083227Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:48.701975547Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.263193725Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.263337559Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.28938162Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.289528534Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.372230976Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.37238677Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 44.314346ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:52.409719932Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 35.80326ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:52.435139088Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.436928364Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.437083428Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 132.567503ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:52.463227382Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 105.825004ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:52.494215601Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.505912012Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:17:52.506786295Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 231.521959ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:52.679333883Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 125.666582ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:52.75677351Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 499.468836ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:52.986836901Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 424.13983ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.050717368Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 66.905736ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.09541133Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 66.400499ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.109165594Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 94.432179ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.238916179Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 139.256193ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.305789903Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 257.417389ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.423065347Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 29.322961ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.544854626Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 938.567636ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.613489016Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 283.260269ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:53.988876093Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.086537261s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.092454423Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 394.169676ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.187125384Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 93.650457ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.225182935Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 402.287492ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.319074596Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 137.87052ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.361175871Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 511.04291ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.527310224Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 879.618349ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.577873268Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 656.145568ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.634099157Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.030662567s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:54.755163328Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 671.437048ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:55.04926522Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.008688978s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:55.189449744Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 936.565387ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:55.302483888Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.103928882s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:55.467275476Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 765.639833ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:55.729721833Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 912.304271ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:55.793910257Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 793.158425ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:56.206959025Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 1.226460018s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:56.248608799Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 596.205241ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:56.506694663Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.02988071s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:56.582093977Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 791.260581ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:56.71839032Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.471198493s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:57.012106872Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 679.182203ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:57.113914707Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 656.5877ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:57.487906318Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.123746364s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:57.521296839Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.078533043s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:57.619656235Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 1.429094971s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:57.800714383Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 799.535364ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:58.152360418Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 607.833981ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:58.490942177Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.407788287s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:58.654027591Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 984.428008ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:58.706185586Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 1.089582971s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:58.96120413Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 548.425541ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:59.10861052Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 628.027279ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:59.240204693Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.220836483s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:59.666341188Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.327103269s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:59.699085172Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 705.200306ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:17:59.877850341Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7\") failed: Property not found. retry in 699.872708ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:00.061654669Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 586.784695ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:00.072027787Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:00.466039104Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46902"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:00.466175398Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"e1498b5b-fdc2-422b-b5d7-51b17e39320d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46902","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":77,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 1.353240934s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:00.592932026Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.455429008s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:00.668757481Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.101954349s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:00.89526793Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.325929755s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:01.034251833Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 1.186711379s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:01.955896073Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 568.163092ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:02.006065906Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.179025208s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:02.136324144Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 551.208448ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:02.373833258Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.467448046s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:02.584879328Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.428327444s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:02.935294192Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 602.157268ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:03.152489161Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.145236426s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:03.324815024Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 1.093600272s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:03.765105365Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.472628669s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:04.062002438Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 762.530317ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:04.373916529Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.322324352s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:04.479240815Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 1.422418747s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:04.867573546Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 851.832451ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:05.146094606Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 732.894569ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:05.545050832Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 1.447640952s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:05.811760497Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.455806159s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:06.008183941Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 870.537679ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:06.287960504Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 821.11926ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:06.299470821Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 1.211228484s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:07.130035782Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 510.851842ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:07.168505574Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 990.192056ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:07.267948408Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.174674042s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:07.472906902Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.263372287s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:07.688929151Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 689.241561ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:08.267503046Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 715.639709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:08.348918545Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 651.892353ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:08.657080769Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 655.754199ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:08.961348184Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 860.26993ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:08.969103103Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce\") failed: Property not found. retry in 1.045985692s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:09.073897415Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 745.536036ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:09.317879865Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5\") failed: Property not found. retry in 1.204386043s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:09.626031429Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60\") failed: Property not found. retry in 911.307892ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:09.839231876Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab\") failed: Property not found. retry in 1.247395107s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:10.072459028Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.343809373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.343877285Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.343883285Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_aef48af3-5864-42c0-a327-7527e34b00ce"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.343887745Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_a283e2ab-09c5-4129-8f26-97e71f66beab"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.343914796Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_d82fdb45-6b3e-42a2-8c59-d538f5bd0b60"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.343935647Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.344386488Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.344755108Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"86c71bb4-1f76-44c8-88b6-f6eb3c2775af","remote_addr":"[fd00:1122:3344:101::1]:34056","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":23166604,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.377170003Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49290"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:11.410689648Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3227"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:26.022272105Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:30.537335412Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:30.538453318Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"755cf514-f991-4626-8a47-4aec98c0f3a2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35119","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1055,"response_code":"200"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.383082598Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.383187967Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"1ae6dcdb-037c-44c7-bc47-edca78ac363d","remote_addr":"[fd00:1122:3344:101::1]:49290","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":22956283,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.417175046Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.417584444Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51895"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.418208646Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.418215912Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.439229559Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.439276294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.460480989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.460524078Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.479402683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.479447262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.49723382Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.497277705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.51622046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.516274208Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.534662227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.534710819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.552506209Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.552547216Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.570780375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.57082366Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.722056904Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.722108938Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.845012127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.845067703Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.952247489Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:34.952301968Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.088347807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.088415421Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.334285583Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.334346626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.486161251Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.486221299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.635170283Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.635224603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.738045666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.738113477Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.895460339Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:35.895516056Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.03764369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.039907317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.157323478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.157437644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: false, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.272045189Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.272144576Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.27253647Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.272681468Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406350281Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406405052Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406413086Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406418467Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406424139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406430656Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406437424Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406444531Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406450081Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406456568Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406460796Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406484525Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406490047Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.4064954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406502095Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406507514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406514367Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406519767Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406526442Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.406654124Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d8d412a0-7104-40bf-b2da-c1258bc1f878","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized)]","latency_us":1983490,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.406898424Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"52.85487ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d8d412a0-7104-40bf-b2da-c1258bc1f878\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d8d412a0-7104-40bf-b2da-c1258bc1f878\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.461165453Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.461900124Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.461913587Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.482144252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.482186675Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.501172412Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.501210101Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.519154497Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.519192036Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.537189602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.537227178Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.554775671Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.554815938Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.571947836Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.571990971Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.588940399Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.588979475Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.606110097Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.606158838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.625731208Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.625773471Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.644959498Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.645001789Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.662601714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.662643125Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.682148964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.682190496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.700109463Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.700159761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.717480081Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.717526151Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.734831078Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.73487216Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.752505272Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.752547892Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.770515833Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.770551984Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.790246571Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.790285404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.810871945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.810918709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.82958289Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.829626475Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.829845706Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.829979509Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.92456356Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924615611Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924624686Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924631417Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924637775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924645837Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924654068Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924660462Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924683722Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924691672Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924699613Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.92470763Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924715936Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924723961Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924729792Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924737143Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924743526Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924751177Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.924759446Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:36.924854793Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3269f59e-dd1d-4c16-bec6-df0038fdd50e","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized)]","latency_us":461876,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:36.925055976Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"161.648922ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3269f59e-dd1d-4c16-bec6-df0038fdd50e\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3269f59e-dd1d-4c16-bec6-df0038fdd50e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.087828276Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.088452839Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.088472574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.108154521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.108212301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.126227851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.12628052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.145955538Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.146006859Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.164656036Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.164706709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.183766369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.183817332Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.200739851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.202965465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.220383435Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.220439181Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.239557012Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.23960433Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.258516794Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.25856142Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.276628293Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.276679932Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.296769464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.296828921Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.315693818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.315743394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.333704639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.333755068Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.352218143Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.352268282Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.37197397Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.372019918Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.391022943Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.391071178Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.409501635Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.40955077Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.42854737Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.428588059Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.44748018Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.447523212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.465389764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.465455091Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.465667075Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.465810103Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558453344Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558501928Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558509203Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.55851524Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558519599Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558544238Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558551541Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558557016Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558563983Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.55857047Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558575692Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558580886Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558587523Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558592389Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558599159Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558604596Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558611187Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558616315Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558620215Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.558697242Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"4535c990-f5ec-4dc0-b466-7ecab4a4a1df","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized)]","latency_us":469146,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:37.558901875Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"205.647238ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4535c990-f5ec-4dc0-b466-7ecab4a4a1df\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4535c990-f5ec-4dc0-b466-7ecab4a4a1df\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.766925147Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.767537416Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.767556007Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.789013279Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.78906639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.809622678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.809670473Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.828551602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.828603137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.848866501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.84890945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.868373945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.868421224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.886977643Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.887024361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.907434733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.907484346Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.924485246Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.924542313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.942428096Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.94247186Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.961968932Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.962022033Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.980475253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.980520647Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.998319197Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:37.998427009Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.015689127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.015733153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.032978706Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.034893062Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.053895128Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.053940935Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.071759253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.071808746Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.09256174Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.092676311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.111282094Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.11132507Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.129696495Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.129805513Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.147334722Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.147388621Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.147581029Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.147714861Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238121229Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238181756Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238218582Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238238083Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238246417Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238254845Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238261774Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238269883Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238279399Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238287621Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238296049Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238304263Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238310656Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238318606Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238326406Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238334507Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238342204Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238348785Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238361451Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.238445398Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"b5ae6134-e553-4d54-8370-93846706ba22","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized)]","latency_us":469795,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.238672419Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"110.359576ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5ae6134-e553-4d54-8370-93846706ba22\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5ae6134-e553-4d54-8370-93846706ba22\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.350138531Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.350816956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.35082964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.37385286Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.373905708Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.395195939Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.395256475Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.416328114Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.416387281Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.437230389Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.437282974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.457659684Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.457711922Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.476974775Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.477035086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.496222764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.496274431Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.514639691Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.514690712Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.53337133Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.533419377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.552293002Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.552344734Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.573888264Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.573999928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.595288667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.595338477Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.61477479Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.614834041Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.63390309Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.633958694Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.652602483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.652651571Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.672567397Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.672621135Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.691185851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.691311478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.709119444Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.709166086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.727134242Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.7271868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.747503496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.747567107Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.747804328Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.747945669Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844036769Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844099283Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844110514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844120762Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844128862Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844136953Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844144913Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844152507Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844160475Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844173891Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844181954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844188395Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844194864Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844199589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844204389Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844218677Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844226834Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844234587Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844239359Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.844287828Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"ebb9a436-d937-449c-9212-ca7874732a9f","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized)]","latency_us":492337,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:38.844505418Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"135.492089ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ebb9a436-d937-449c-9212-ca7874732a9f\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ebb9a436-d937-449c-9212-ca7874732a9f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.982317588Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.982965604Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:38.982980765Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.005194256Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.005253227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.026879736Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.026931375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.048040379Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.048096134Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.068065494Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.06811566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.087049611Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.087114394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.106330807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.106450729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.125316676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.12537409Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.142970958Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.143020047Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.16076911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.160823692Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.178427749Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.178480139Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.198701844Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.198759041Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.217803391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.217850755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.235498904Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.235556299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.254949324Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.255002097Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.272898423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.272949003Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.290749206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.290804492Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.310252067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.3103032Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.329723353Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.329777121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.347523575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.34757077Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.367245386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.367324354Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.367508906Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.367640572Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463213952Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463283704Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463293744Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463366807Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.46337532Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463384085Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463392589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463406005Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463412999Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463421634Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463429762Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463437018Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463454522Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463463101Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463471369Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463479469Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463486304Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463494019Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463502045Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.463623632Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8f6b38e1-8960-490f-955a-2e4c567d08aa","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized)]","latency_us":479561,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:39.463874711Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"371.429481ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f6b38e1-8960-490f-955a-2e4c567d08aa\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8f6b38e1-8960-490f-955a-2e4c567d08aa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.837152975Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.838100381Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.838119516Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.861265334Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.86131866Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.882747068Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.882797891Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.903019056Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.903066617Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.923026216Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.923085205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.942277358Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.942325846Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.961874988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.961928164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.981659678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.98178093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.999501327Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:39.999550698Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.017949238Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.018002386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.03808275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.038131774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.057217056Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.057274075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.077152819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.077216618Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.096317243Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.096367493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.116179558Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.116233297Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.136210682Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.136258909Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.155444633Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.155491818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.173714758Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.173770822Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.194950908Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.195008818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.214816379Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.214879184Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.233872404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.233928608Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.234143976Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.234278773Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330536605Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330587944Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330598154Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330606948Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330616557Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330622895Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330630799Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330637286Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330711254Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330728691Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330737308Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330744189Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330750555Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330755702Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330760577Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330765405Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.33077329Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.33077839Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.330784615Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.330848311Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"82f95dfe-488d-4216-9a04-2798923175c9","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized)]","latency_us":491571,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:40.331087351Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"251.099979ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"82f95dfe-488d-4216-9a04-2798923175c9\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"82f95dfe-488d-4216-9a04-2798923175c9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.584162476Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.58482287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.584842558Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.606888629Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.606955802Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.627387109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.627437519Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.647249164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.647300944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.667967235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.668023178Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.687568255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.687678194Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.706338973Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.706383786Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.725781238Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.725827233Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.744348986Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.744403831Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.764741105Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.764787672Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.785261574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.785307214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.80426726Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.80432938Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.822896518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.822942091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.841536626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.841586962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.860332586Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.860381994Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.879372504Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.879432711Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.898389476Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.898436944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.91751115Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.917556395Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.936072225Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.936116992Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.954517194Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.954578911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.973045069Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.97311151Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.973335861Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:40.973545316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069763986Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069833287Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069848175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069854269Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069869616Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069875626Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069891442Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069898924Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069905692Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069911383Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069918424Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069923693Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069929384Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.0699335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.06993769Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.06994174Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069947196Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069952718Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.069956665Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.070016386Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"dd4edc92-cbd2-45e0-a3a9-ed5652c1514c","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized)]","latency_us":484062,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.070244476Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"275.709859ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dd4edc92-cbd2-45e0-a3a9-ed5652c1514c\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dd4edc92-cbd2-45e0-a3a9-ed5652c1514c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.347141797Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.347766862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.347776677Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.36841357Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.368541563Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.38835997Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.388415096Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.408349493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.40841932Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.427469848Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.42752804Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.446622318Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.446749531Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.466313709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.466368207Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.484854481Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.484925818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.504306916Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.504364762Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.524012051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.524070778Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.542282743Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.542348576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.5622582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.562326311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.581563708Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.581628622Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.599370288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.599423248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.618795411Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.618861122Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.636658205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.636713228Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.656583018Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.656648795Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.675484457Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.675535795Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.695052584Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.69511956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.715244284Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.715301361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.733750091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.733837777Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.734073781Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.734221141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.829984584Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830069965Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830081843Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830107803Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830114141Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.83012165Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830127622Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830133612Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830141225Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830150526Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830159723Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830173401Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830181145Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830188927Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830196286Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830203627Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830209608Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830217352Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830236215Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:41.83024245Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c46328f1-ad55-490d-8952-ccef1aa49daa","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized)]","latency_us":481326,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:41.830475781Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"424.735844ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c46328f1-ad55-490d-8952-ccef1aa49daa\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c46328f1-ad55-490d-8952-ccef1aa49daa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.258138174Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.258775962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.258790372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.281373905Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.281435518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.302399573Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.302453068Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.321142841Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.321192136Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.340824397Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.340884735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.360748753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.360799257Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.379503742Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.379565262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.398931962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.398977057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.416605013Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.416656971Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.437153881Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.437207854Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.456254548Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.456305895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.474246245Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.47430006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.492817904Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.4928786Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.51208146Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.512124473Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.531256522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.531312172Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.551399184Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.551448967Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.570869323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.570933833Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.592096591Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.592172709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.61260827Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.612673184Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.632544881Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.632599295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.651607421Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.65169147Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.651889194Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.65211417Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.74590238Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.745996217Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746007093Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746018118Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746027671Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.74603668Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746047987Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746057175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.74606624Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746084419Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746090269Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746097807Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746110754Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746117139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746255653Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746281875Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.74629622Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746304198Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746309814Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:42.746317304Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"91dfad5c-ea6f-48df-a800-cd9a481d88a9","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized)]","latency_us":486309,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:42.746541467Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"585.269487ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91dfad5c-ea6f-48df-a800-cd9a481d88a9\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91dfad5c-ea6f-48df-a800-cd9a481d88a9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.334644101Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.335405423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.335419842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.358488757Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.358556221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.378450962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.378507626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.396827788Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.396884275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.416100072Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.416160842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.434200045Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.434262859Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.454443995Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.454497321Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.471987359Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.472040705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.49089148Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.490953807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.509870522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.50993328Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.530136787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.53018701Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.548836743Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.548891729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.56801202Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.568060996Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.586921049Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.58697655Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.60485945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.604911069Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.624213663Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.624271996Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.642146625Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.642197963Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.661216422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.661266646Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.678991329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.67904712Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.698737075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.698853286Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.717607905Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.717706458Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.717955135Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.718097629Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811197126Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811269972Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811282534Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811291197Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811299925Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811308344Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811330057Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811344055Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811349408Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811356149Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811360864Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811365777Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811391409Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811398093Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.81140294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811409343Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811421635Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811428282Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811432932Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:43.811445251Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d0e822cf-b2e7-404f-af3b-ecc26e75d23c","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized)]","latency_us":474970,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:43.811716008Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"465.37291ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0e822cf-b2e7-404f-af3b-ecc26e75d23c\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d0e822cf-b2e7-404f-af3b-ecc26e75d23c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.279485253Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.280166701Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.28018337Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.303013808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.305491072Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.322501553Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.322560326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.342039545Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.342099996Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.360272771Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.360330147Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.378651999Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.37871876Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.396092631Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.396144899Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.414646385Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.414767167Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.433351576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.433403806Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.453400366Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.453457667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.471988743Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.472041741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.491481284Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.491533147Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.509801552Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.509864516Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.52825809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.528315888Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.546832647Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.546879476Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.564868386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.564913987Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.583441403Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.58349126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.601442408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.601501979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.621300986Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.621355005Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.638732116Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.638798932Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.657265291Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.657408907Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.657620525Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.657772422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749250312Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749313267Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749327029Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749334708Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749344955Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749352549Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749358136Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749365149Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749370803Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749378059Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749398816Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749406288Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749413104Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749418935Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749425816Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749430298Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749435567Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749439663Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749448805Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:44.749521435Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8709df97-f24a-4340-b34e-61d024d8db6c","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized)]","latency_us":468289,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:44.749832737Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"748.410377ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8709df97-f24a-4340-b34e-61d024d8db6c\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8709df97-f24a-4340-b34e-61d024d8db6c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.501191167Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.501889574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.501920616Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.524518329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.52457564Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.545823263Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.545900223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.565120573Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.565186359Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.584473644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.584553306Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.603882662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.604005271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.62260043Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.622670518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.642490813Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.642561109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.662664471Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.662717216Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.680938976Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.681001687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.701226775Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.701292889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.719988993Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.720044026Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.738602109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.738653036Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.757427322Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.757493877Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.776812455Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.776861956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.796533185Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.798965556Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.815526749Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.815594344Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.834640227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.834697576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.853056618Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.853114895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.871347887Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.871391622Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.890343076Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.890406856Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.890607336Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.89080854Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988465991Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988539399Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988549046Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988567966Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988576197Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988581728Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988588197Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988611392Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988617992Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.98862551Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988630817Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988635608Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988643352Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988648264Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988654592Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988659186Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988664108Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988672067Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988678387Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:45.988683047Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c857b958-e99d-40ca-a88d-1738ad1eb0b1","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized)]","latency_us":485716,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:45.988916902Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"664.360023ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c857b958-e99d-40ca-a88d-1738ad1eb0b1\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c857b958-e99d-40ca-a88d-1738ad1eb0b1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.656447017Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.65713811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.657146107Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.680055089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.68011211Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.701406885Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.701458017Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.719647708Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.719717075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.739092465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.739139191Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.758343792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.758389628Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.77765896Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.777704608Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.796285608Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.796348291Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.814948571Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.815002104Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.83435988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.834407329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.855135386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.855187409Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.875939948Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.875989374Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.896990462Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.89703761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.916265016Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.916311076Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.935421393Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.935472798Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.95405715Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.95411013Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.973250989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.97330499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.992633974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:46.99275729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.011142529Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.011200186Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.030124283Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.030172369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.053114409Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.053185004Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.053378182Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.053510119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.147880863Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.147967124Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.147980175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.147999703Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148031729Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148043543Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148053255Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.14812836Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148141298Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.14814741Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.14815717Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148164614Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148173399Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148186674Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148194465Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148207037Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148214734Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148220143Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.14822801Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.148239335Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"436304f8-7d22-42ee-994d-2e6392b689ec","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized)]","latency_us":489956,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:47.148466459Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"726.654038ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"436304f8-7d22-42ee-994d-2e6392b689ec\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"436304f8-7d22-42ee-994d-2e6392b689ec\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.87774074Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.878834823Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.878851371Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.90148177Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.903908181Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.921442733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.92149785Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.941322141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.941374764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.960794821Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.960848588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.97978868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.979844341Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.998338045Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:47.998467819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.017908546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.01797781Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.036188164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.036242408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.055966383Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.056018481Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.075010545Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.075063853Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.094249668Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.094313298Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.112828598Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.112881681Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.131984154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.132038249Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.151568153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.151634239Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.170428483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.170484687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.189764988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.192140563Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.207729998Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.20777514Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.225466453Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.225524101Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.243731478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.243786164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.263354666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.26343279Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.263617585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.263759395Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360459851Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360584035Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360597508Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360607661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360620008Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360629599Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360645425Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360653168Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360664325Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360672304Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360681651Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360691204Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360697616Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360705332Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360712505Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360719873Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360731714Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360738043Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360747184Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:48.360754534Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f2a86685-7c54-4d80-9dbe-96846d148533","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized)]","latency_us":480915,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:48.360989327Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.920639626s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2a86685-7c54-4d80-9dbe-96846d148533\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2a86685-7c54-4d80-9dbe-96846d148533\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.287876615Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.288473592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.288490768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.31140897Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.311471511Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.33200177Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.33207034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.354621087Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.35467566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.372984755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.373039197Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.392972193Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.393022567Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.411959477Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.412015925Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.430386648Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.430464743Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.448902118Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.448954582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.467469903Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.467540349Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.487194965Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.487245525Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.505853317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.505910234Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.524298952Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.52436875Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.543110647Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.543173649Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.561761877Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.561826378Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.579819438Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.579889528Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.598215626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.598269495Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.617919268Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.617971648Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.636347123Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.636408822Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.655019405Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.655078048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.673033667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.673097325Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.673292002Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.673420227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.76599861Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766075496Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766085472Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766094144Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766102516Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766110541Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766125073Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766133436Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766141648Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766148426Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766158814Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766165537Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766173205Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766189546Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766200919Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766206094Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766214034Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766218694Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766225013Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:50.766290171Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"10ba8ad7-44a8-4938-809b-693b91ffcb1f","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized)]","latency_us":476691,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:50.766528743Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"840.52751ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10ba8ad7-44a8-4938-809b-693b91ffcb1f\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"10ba8ad7-44a8-4938-809b-693b91ffcb1f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.610070791Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.611285969Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.61129451Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.633743055Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.633807575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.654185956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.654304253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.672231721Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.674686026Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.691836803Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.691886568Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.710676281Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.71073617Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.728343013Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.728399807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.747853422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.747906252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.766286917Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.766336006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.785049066Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.785107088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.804493652Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.804560377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.823942923Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.824001865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.842394901Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.842454443Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.862408468Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.862460201Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.880451291Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.880506465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.899637169Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.902003715Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.918622748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.918681962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.936717897Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.936772696Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.95525349Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.955302757Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.973460734Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.973526886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.992271141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.992339496Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.992512451Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:51.99263251Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086075398Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086134199Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086142224Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086151469Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.0861693Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086175967Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086184329Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.08619107Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086205873Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086214526Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086229452Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.08623609Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086241255Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.08624644Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086252571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086261178Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086274312Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086279206Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086284045Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:52.086323814Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c68a700f-e226-4f2c-b788-c25ccd33dd8b","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized)]","latency_us":474047,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:52.086615674Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.829244461s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c68a700f-e226-4f2c-b788-c25ccd33dd8b\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c68a700f-e226-4f2c-b788-c25ccd33dd8b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.922274743Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.922954028Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.922969084Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.945143703Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.945203377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.96356842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.965854377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.981585341Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.981636624Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.999427042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:53.999506413Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.02078633Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.020836441Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.039878352Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.039934678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.060071811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.060135807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.079360767Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.079419456Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.099445176Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.099504989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.118984229Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.121398498Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.137935056Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.13805718Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.156842474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.156905092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.17518501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.175244487Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.193312195Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.193370968Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.213048375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.21310307Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.232124599Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.232186607Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.251589269Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.251642933Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.271745739Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.271796422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.291484031Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.291551111Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.310388667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.310536102Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.310771965Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.3109332Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.403819589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.403927695Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.403945367Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.403959139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.403972771Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.403985718Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404006175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.40402351Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404033147Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404043658Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404054326Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404062567Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404071446Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404087383Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404097087Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404104072Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404110775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404117225Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404124341Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:54.404130959Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"2a5d8e2a-1857-4e1d-9809-5d0cfc89c69e","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized)]","latency_us":480054,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:54.404382403Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.085333464s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a5d8e2a-1857-4e1d-9809-5d0cfc89c69e\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2a5d8e2a-1857-4e1d-9809-5d0cfc89c69e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.496738628Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.497879814Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.497889704Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.521359878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.521494426Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.541939318Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.541997417Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.561873247Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.56193172Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.580661313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.580713112Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.600091786Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.602936393Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.622624625Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.622750552Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.646389318Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.649175121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.669234086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.669308235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.692614668Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.692671425Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.716310917Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.71637046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.740485288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.740561583Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.764132404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.76419338Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.787269657Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.787334825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.810596265Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.812938757Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.834211396Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.83433894Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.858073676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.858131812Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.882278016Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.882327509Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.905656338Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.905733993Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.930465652Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.930597698Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.954503238Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.954651656Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.954885333Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:56.95504851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069104426Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069201648Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069219751Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069229783Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069243199Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069266628Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069276134Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069283962Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069295671Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069305741Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069316813Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069324857Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069335883Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069343139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069352411Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069358223Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069363698Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069372999Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069380584Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:18:57.06938649Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"aa6fc854-c187-44eb-a851-5623764fd90c","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized)]","latency_us":570278,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:18:57.069666941Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.936171308s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa6fc854-c187-44eb-a851-5623764fd90c\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:18:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aa6fc854-c187-44eb-a851-5623764fd90c\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:00.652809097Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:00.652918431Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"717df959-6f08-4842-bf9e-1e09f809abb8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42142","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":65,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.017808699Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.019225816Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.01923521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.041121025Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.041176874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.060951011Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.061016095Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.079204572Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.079271464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.097474618Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.097526979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.115782424Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.115901929Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.133242748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.133299432Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.151353748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.15140354Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.169682098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.169735386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.188299925Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.188347785Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.208225714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.208276236Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.227617875Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.227668266Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.248349521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.248391364Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.268429223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.268482962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.288054686Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.288120613Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.308023566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.308081664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.328343236Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.32847865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.347699645Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.347753627Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.367032073Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.367081463Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.386146682Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.386201086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.405907804Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.405968715Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.406172299Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.406302633Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.495975279Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496064128Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496082401Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496091382Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496102398Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496110142Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496118308Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496130449Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496140058Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496148093Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496154974Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496161959Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496167734Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496174578Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496196338Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496202057Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496206107Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496210279Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.496214375Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:01.496277818Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1246e861-ff94-4dc0-9011-0965a0852966","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized)]","latency_us":476094,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:01.49649282Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.683364449s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1246e861-ff94-4dc0-9011-0965a0852966\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1246e861-ff94-4dc0-9011-0965a0852966\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.185751396Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.186361143Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.186378168Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.208614732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.211317648Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.229390737Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.229437726Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.24801845Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.248071298Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.267581685Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.267698468Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.285353429Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.285427315Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.303643035Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.303688843Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.322352512Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.322394447Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.341291031Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.341351343Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.35948236Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.359525064Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.378120925Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.378166705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.396047Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.396092179Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.413945366Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.414064173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.433579055Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.433622846Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.45183518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.451882469Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.469345479Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.469396124Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.488168306Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.488299839Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.506649104Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.506701128Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.524944321Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.525006779Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.543558631Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.543619175Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.561975914Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.562040397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.562237024Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.562361331Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653141848Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653206284Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653217291Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.6532338Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.65324567Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653255654Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653265245Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653274048Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653283161Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653294317Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653300561Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.65332774Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653337753Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653345816Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653351469Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653366338Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653373932Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653379557Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.653387226Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:03.653402011Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"dc58ee13-30cc-4cb0-9b1a-52757c906a73","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized)]","latency_us":465956,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:03.65372286Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"5.170829824s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dc58ee13-30cc-4cb0-9b1a-52757c906a73\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dc58ee13-30cc-4cb0-9b1a-52757c906a73\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.839995866Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.840614782Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.840629088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.863968584Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.864031024Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.884586565Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.884655942Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.903502027Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.903557173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.921324566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.921380705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.93873388Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.938795673Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.958033967Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.958110019Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.97692311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.976993968Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.99514124Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:08.995203858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.013848091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.013905092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.032504193Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.032558495Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.051133535Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.051188746Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.069206472Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.069343889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.086859323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.086920918Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.105521344Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.105579123Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.124183753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.124256589Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.142766499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.142830147Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.160837231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.160898105Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.179117988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.179172701Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.197715664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.197777025Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.215482879Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.215575797Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.215891967Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.216075188Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307511792Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307574015Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307581656Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307596797Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307603894Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307614367Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307620714Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.30762901Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307654427Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307662799Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307675821Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307680847Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307685675Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307690456Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307696672Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.30770915Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307715919Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307720935Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.307729035Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:09.307762936Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c66ea20f-45aa-45dd-846e-d0548ad633c5","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized)]","latency_us":466097,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:09.308058694Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.978218887s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c66ea20f-45aa-45dd-846e-d0548ad633c5\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c66ea20f-45aa-45dd-846e-d0548ad633c5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.297953719Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.299171759Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.299188878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.321589145Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.321659713Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.341983297Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.342048736Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.361130232Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.36119583Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.380120208Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.382459094Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.398129705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.398202851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.415991026Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.416125546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.434486806Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.434555808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.451867824Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.451916613Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.470111839Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.47023786Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.488751493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.488802109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.506689826Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.506742094Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.524953995Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.525019518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.542547702Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.542609015Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.560672928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.560727464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.57908199Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.579167492Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.597263676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.59732198Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.615461822Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.615541802Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.634355891Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.634414871Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.652624489Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.652747771Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.670991414Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.671047994Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.671278831Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.671435023Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764674016Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764737111Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764746965Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764752721Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764758458Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764782637Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764790869Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764798116Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764805147Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764810762Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764817842Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.76482462Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764830123Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764834238Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764840051Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764844101Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764848685Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764854114Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.764859467Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:13.764916853Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a6a1cdeb-7272-424d-87d4-39c30b12f3e5","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized)]","latency_us":465229,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:13.765179492Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"6.631404321s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a6a1cdeb-7272-424d-87d4-39c30b12f3e5\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a6a1cdeb-7272-424d-87d4-39c30b12f3e5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.415830937Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.416643914Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.416668749Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.438220549Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.438275928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.457660095Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.457714687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.477650783Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.47996401Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.497238725Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.497295369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.51582083Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.515887075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.534888919Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.534954903Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.554899252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.554957838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.573072684Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.573128944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.59121768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.591273229Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.609678807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.611993389Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.628586808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.62863796Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.646823915Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.646884957Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.664747306Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.664811874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.682441045Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.682509953Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.700322006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.700381492Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.718254669Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.718311643Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.73674316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.736867076Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.754769422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.754822767Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.772733889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.772778384Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.790846618Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.79100807Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.791202081Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.791345428Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883728242Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883798332Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883824985Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883838973Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883847767Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883856551Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.88386332Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883870155Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883877459Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883886027Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883899612Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883915025Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883921597Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883928235Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883936456Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883944876Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883951139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.883962773Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.88396971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:20.884076373Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"0ffcd191-65e5-4d71-8dcc-14f8186afe33","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized)]","latency_us":466411,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:20.884822304Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"8.810442913s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ffcd191-65e5-4d71-8dcc-14f8186afe33\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ffcd191-65e5-4d71-8dcc-14f8186afe33\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.719888255Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.720622556Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.720634565Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.743724629Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.743788015Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.764726768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.764797944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.783945224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.78400667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.804182721Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.804247317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.822613614Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.822669808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.841787714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.841843768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.861222189Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.86128685Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.880708767Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.880764513Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.901202418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.901274983Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.921359864Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.921429213Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.941371595Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.941429533Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.962569287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.962632429Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.982048134Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:29.982117032Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.003590957Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.006221285Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.023649464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.023708819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.043048025Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.043171733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.062588141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.062651996Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.083694962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.083752057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.104720374Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.104786592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.125013733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.125092016Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.125362854Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.12551672Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.222984234Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223057051Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223067158Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223075343Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223081202Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.22308769Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223093286Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223100909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223108034Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223113715Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223119331Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.22312625Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223132091Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223139385Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223155801Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223170013Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223174167Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223179642Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223183739Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.22325245Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"63faa4e0-cfbc-40c6-b81b-d153639536d3","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized)]","latency_us":501513,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:30.223474494Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"8.360929158s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63faa4e0-cfbc-40c6-b81b-d153639536d3\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63faa4e0-cfbc-40c6-b81b-d153639536d3\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.766941461Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:30.767016322Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5380a5f0-17b4-45a1-a9a5-a7bc513d4ac8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52151","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.608761951Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.609476074Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.609486162Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.633926119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.633984378Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.655331228Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.655386017Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.674907197Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.674962745Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.694220546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.694268322Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.713190765Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.713256992Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.731078716Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.731153961Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.749101808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.749157103Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.766616368Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.766687123Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.785331601Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.785382414Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.803594321Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.803643006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.822178255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.822239232Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.840181015Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.840235381Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.858407586Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.858472021Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.876770882Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.876814992Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.894721478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.894788286Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.912487801Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.912547615Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.930571664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.930620386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.948987109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.949045253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.967703691Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.967762577Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.985953046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.986041332Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.986261838Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:38.986474997Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082110504Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082217157Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082241196Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082266227Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082280609Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082308546Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082319282Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082326294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082339419Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082344614Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082357598Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082366083Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082372589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082377436Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082383661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082389943Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082396027Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082412762Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.082419653Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:39.08242435Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"65e9e53b-bc35-4fa7-8fb2-4c4bc0d1a20f","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized)]","latency_us":471898,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:39.0826487Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"7.792290225s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"65e9e53b-bc35-4fa7-8fb2-4c4bc0d1a20f\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"65e9e53b-bc35-4fa7-8fb2-4c4bc0d1a20f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.897104002Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.897800263Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.897814871Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.920471695Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.920543762Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.94198193Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.94204796Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.961613417Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.961673419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.979797376Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.97984744Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.998184344Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:46.998311758Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.016561738Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.016623708Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.036773014Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.036823585Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.055938429Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.055984697Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.074345042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.074401491Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.093789474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.093840756Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.112920598Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.112997653Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.130900003Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.130951575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.149334725Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.149384554Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.167199419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.167248367Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.185627611Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.185669369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.203307963Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.203367823Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.221157025Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.221209508Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.238640234Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.238684496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.257366818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.257410516Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.276540969Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.276615446Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.276834386Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.276964832Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368547274Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368623007Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368634632Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368651189Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368659224Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368678612Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368689065Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368700334Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368718812Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.36872877Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368736673Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.36874632Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368763204Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368773049Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368782658Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368798447Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368806293Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368812143Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.368818012Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:47.368907395Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"9bec3d89-b808-40e7-b95a-ed9b81ca87ce","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized)]","latency_us":469997,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:47.369190603Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"9.358363853s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9bec3d89-b808-40e7-b95a-ed9b81ca87ce\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9bec3d89-b808-40e7-b95a-ed9b81ca87ce\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.754991632Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.75614325Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.756161158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.779223399Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.781669529Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.800617444Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.800684046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.820115526Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.820183393Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.839400848Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.839452289Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.857591014Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.857661431Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.875716053Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.875771517Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.894091764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.89421248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.912306252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.912359344Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.930190833Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.930313823Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.948110635Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.94818606Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.966593732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.966648952Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.984513394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:56.984567631Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.004650823Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.004703718Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.023703235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.023764484Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.043467824Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.043535588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.062552803Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.062671857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.081546775Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.081603233Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.101622069Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.101677553Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.120908459Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.120962797Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.139170351Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.139234347Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.139463449Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.13960314Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.230929008Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.230986909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231009476Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231015888Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231022048Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231029642Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231034217Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231044895Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231051449Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231057103Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231128957Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231137947Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231149966Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231162126Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231169833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231175411Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231224771Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231235685Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.2312488Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:19:57.231330815Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"e7459d9e-b5e9-4bad-8371-2bb5d1cccbce","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\", TimeNotSynchronized), (\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\", TimeNotSynchronized), (\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\", TimeNotSynchronized), (\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\", TimeNotSynchronized), (\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\", TimeNotSynchronized), (\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\", TimeNotSynchronized), (\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\", TimeNotSynchronized), (\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\", TimeNotSynchronized), (\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\", TimeNotSynchronized), (\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\", TimeNotSynchronized), (\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\", TimeNotSynchronized), (\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\", TimeNotSynchronized), (\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\", TimeNotSynchronized), (\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\", TimeNotSynchronized), (\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\", TimeNotSynchronized), (\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\", TimeNotSynchronized), (\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\", TimeNotSynchronized), (\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\", TimeNotSynchronized), (\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\", TimeNotSynchronized)]","latency_us":474254,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:19:57.231537464Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"13.890851378s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e7459d9e-b5e9-4bad-8371-2bb5d1cccbce\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:19:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e7459d9e-b5e9-4bad-8371-2bb5d1cccbce\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:00.883238639Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:00.883361418Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a5a53930-b641-4151-9bac-047a779008dc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36783","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":68,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 9e69a966-aaa5-463c-adce-1c59004ff150, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 407bc712-4fe0-4054-93d2-8de0fc6a6fdd, kind: Service { id: 01d1c8e5-8905-4e34-824f-8a30fe31e240 }, name: Name(\"ntp-01d1c8e5-8905-4e34-824f-8a30fe31e240\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a283e2ab-09c5-4129-8f26-97e71f66beab, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: aef48af3-5864-42c0-a327-7527e34b00ce, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: de116760-7ce8-4051-89b5-dbe725a06843, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 0ed061a1-d644-4f36-95d9-99c559521914, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b05778b9-9d7a-46ab-9333-eb3104c321fe, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2063053f-6bdd-4615-8127-179923ae9a65, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 406ab4c7-818a-4298-ae6e-64e3dfad5143, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 00b7bf82-35cc-48fe-9476-270d45e6d301, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6ebf9979-ce51-4c91-a9a2-011bb6255058, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: a7894346-dd44-48fa-aeb3-3d055054a846, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: c263e526-2219-421f-a41f-871118e2593a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: a839add2-c178-4fc2-ac03-7efdcd3ad938, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6e567479-af48-4e4b-925c-59aa4134ba69, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.160814561Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.161594287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 842e2f1e-ea91-42d4-acc8-64d93d1c76c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.161606746Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.183413355Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1f905b68-a2ec-4d9a-b15d-e713ba321080, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.183469869Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.202507034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9e69a966-aaa5-463c-adce-1c59004ff150, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.20256578Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.221922808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d82fdb45-6b3e-42a2-8c59-d538f5bd0b60, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.221972028Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.239942451Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 72e08aa2-1eb6-451a-a9af-6ce17d6fe3e5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.239991942Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.258130639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a283e2ab-09c5-4129-8f26-97e71f66beab, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.258191025Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.276083078Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: da4a9f24-d8c8-4dcc-85d4-53a41bf39cd7, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.276128829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.294145629Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: aef48af3-5864-42c0-a327-7527e34b00ce, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.294195712Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.312008921Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: de116760-7ce8-4051-89b5-dbe725a06843, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.312072287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.330660053Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0ed061a1-d644-4f36-95d9-99c559521914, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.330713033Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.34894589Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 00b7bf82-35cc-48fe-9476-270d45e6d301, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.348997453Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.367659865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c36baa1a-e79a-4644-8a6b-9dc975d1f079, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.367708587Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.386350521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc500983-cf20-4c4d-8f8c-f5db945b3e2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.38640547Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.404870633Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca7c0b15-dbe1-4587-87c3-9086e4f965d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.407257406Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.423531919Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a839add2-c178-4fc2-ac03-7efdcd3ad938, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.423587833Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.441893318Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 49abcdd7-0bae-4ae0-88f1-0467b03df954, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.441943091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.459988304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.460048438Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.47960515Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e567479-af48-4e4b-925c-59aa4134ba69, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.47965346Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.497749835Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.497801305Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.515429079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: c803c23d-6aa1-46ff-a7bf-b16818f7abe5, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.515479134Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.533486952Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.533570054Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.533883956Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.534103778Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.623314567Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.637081764Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.637412748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.638781499Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.640163307Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.675339799Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.675785476Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.675824065Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.675845872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.689040769Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.689123355Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.881212551Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.881328823Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.881336482Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.881342745Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.88135153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.881357061Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.881368087Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.881375437Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.882186703Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.888454358Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.893559909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.898620045Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.903679864Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:11.908422206Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.064427161Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.071065919Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.076077765Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.080709721Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.085985403Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.09078137Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.09531542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.100106554Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.104894694Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.110543244Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.115951933Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.120941662Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.126163502Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.131956756Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.189088466Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.230674922Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.23133539Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.23799339Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.243219163Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.248337111Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.253437283Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.258134841Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.263130199Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.268079652Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.313228661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.371155808Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.403345276Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.405654507Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.405669292Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.480047449Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 234, 191]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.480093023Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.484292224Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.484302771Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.543598358Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 178, 44]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.54364335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.547402412Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.547421013Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.60207612Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 242, 2]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.602128031Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.605727108Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.605737993Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.661872314Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 188, 245]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.661942938Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.664191702Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.664212075Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.732197491Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 188, 32]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.732233277Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.732905847Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.732914772Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.733229265Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.733235359Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.733549854Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.736837313Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.736866188Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.740060257Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.740075407Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.740080019Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.765296229Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.765418813Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.788916944Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.791782483Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.791808527Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.791823527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.792207678Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.801015018Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Configuring new Omicron zone: oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.804171686Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.839234857Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.841615347Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.841655896Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Installing Omicron zone: oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.841795296Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.87222644Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.878083092Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.909101545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.913717921Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:12.950958424Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:13.013140429Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:13.038502714Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.017870859Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.053476117Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.053494387Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.146135435Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.148546908Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.149463181Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.15058755Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.15096431Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.158955066Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.159749326Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.191484335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.192200533Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.194183654Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.194775859Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.195940739Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.202622187Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.209053837Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.214486157Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.220554064Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.251934753Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.258232195Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.264160888Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.270910602Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.297273922Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.473047333Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:24.473619488Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 30.622852ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.451741902Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 62.418719ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.461987956Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 88.207731ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.491369434Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 149.999845ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.536528698Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 55.48336ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.545952891Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 50.380172ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.558895044Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 68.365371ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.568358739Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 44.535693ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.577367701Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 39.252703ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.586787774Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 35.114169ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.595118708Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 48.387761ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.602634372Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 48.517694ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.610129935Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 73.196185ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.617349152Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 64.517149ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.627194445Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 32.308442ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.636640369Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 112.6039ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.645389144Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 57.621346ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.653873823Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 173.896485ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.665697398Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 95.934788ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.676092056Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 120.891828ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.692440017Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 115.734044ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.73636268Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 293.701568ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.747907757Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 70.613411ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.758411248Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:25.759864106Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 86.024707ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.769486834Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 75.703925ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.794155239Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 73.512026ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.850345628Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 288.960729ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.930646348Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 115.324867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:25.98853135Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 62.377465ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.032795532Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:26.033002877Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 227.698613ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.092122611Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 259.067764ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.12426353Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 267.753332ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.139850517Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 260.278917ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.169038904Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 100.50183ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.185275342Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 178.781596ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.201355107Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 106.576939ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.215218374Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 296.943622ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.226637729Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 108.323362ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.244404157Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 588.829316ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.255270657Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 122.189588ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.268802736Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 523.140811ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.280877737Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 443.378915ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.293697347Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 229.30824ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.304814064Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 425.135466ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.333154185Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 287.833613ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.344735813Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 308.820449ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.365293893Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 69.483347ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.382403744Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 242.252133ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.391196511Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 218.594564ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.400635274Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 582.494527ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.410474788Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 599.40294ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.418983677Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 290.251205ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.442247487Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 133.488127ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.461708749Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 221.613504ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.541202558Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 557.301739ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.55834048Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 151.073338ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.621248492Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 649.862698ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.650349142Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.183905417s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.738136445Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 499.239438ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.77944233Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 559.12619ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.79381705Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 202.526479ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.812283286Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 1.195425489s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.824799339Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 565.21266ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.836244754Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 835.589469ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.852196045Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 922.195973ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.861005742Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.135892256s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.880362231Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.159751121s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:26.894947528Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 632.903677ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.00753403Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.158300726s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.030657706Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 480.111543ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.040312855Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.364825746s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.129919185Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 689.80507ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.292563118Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.248756958s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.314367389Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.264104734s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.477955537Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 631.453394ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.625675065Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 774.541213ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.775470027Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 606.197919ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.848642333Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 782.424427ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.897553214Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.177308085s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:27.9660428Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 759.71806ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.026652942Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 946.85741ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.103172345Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.284206508s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.127662116Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 684.086351ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.206533709Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.221804078s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.420754352Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 563.430726ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.509544931Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 1.09582336s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.575120411Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 916.309709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.624172956Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 972.103285ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.660084032Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.152079336s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.724261156Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.183037595s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.761476565Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 1.216884893s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.824751846Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 562.488026ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.900867039Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.322572878s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.95445538Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 886.634863ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:28.96918653Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:29.007314913Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:29.048348871Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 546.116783ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.120411248Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 1.306035961s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.190369922Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:29.190943277Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.374550402s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.277211971Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 30.762897ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.343846919Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 26.924111ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.377986259Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 104.166092ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.417939518Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 55.431026ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.454581663Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 552.01799ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.491675489Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 978.073512ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.527105293Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 49.932222ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.5580252Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 280.990819ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.606650954Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 236.353247ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.62009476Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 761.668362ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.66702197Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 889.56056ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.717703796Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 66.859696ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.745158294Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.049085002s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.758095788Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 1.49098425s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.791611112Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 588.73705ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.814154623Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 231.251866ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.832396103Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 419.949013ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.911530953Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 723.511924ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:29.932966316Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 349.214399ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.003277778Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.102815891s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.087690804Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 361.86358ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.098402301Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.194570603s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.108000298Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.008021418s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.117495923Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 570.963422ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.127970093Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 806.431759ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.291091328Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 910.356212ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.345885321Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 494.799942ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.366301827Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 925.030547ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.416042929Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 1.383969742s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.440741456Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 1.106542085s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.473914621Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 1.291974847s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.5114769Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.258589068s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.526295831Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 629.750833ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.621457865Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 718.215099ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.664440653Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.05104212s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.67597711Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 886.93933ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.711083215Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 960.694991ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.821370038Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.465632087s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:30.874661272Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.247646476s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.109610659Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 638.737682ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.136066161Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.227463048s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.204747222Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.092381838s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.266316099Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 1.066130274s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.278921954Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 1.014822457s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.295198244Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 548.336309ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.315782534Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 1.116958352s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.356788921Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.362648102s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.395415167Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 1.476960302s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.595365572Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 760.292514ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.611128578Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:31.654413714Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54141"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:31.721241797Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"f28eb472-e5b1-4853-9d84-6bc1ffbb936b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54141","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":35298,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.157094565s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.767653943Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.035445224s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.788324226Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.255970226s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.800324606Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 877.300693ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.81177399Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 559.337653ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.829278862Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 1.480711664s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.84744668Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.1988836s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:31.878779178Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 728.499178ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.324938379Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 765.872779ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.354985944Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 1.315314654s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.36645262Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.394564909s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.37888894Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 954.848282ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.390021207Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 876.482462ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.426151079Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 1.30526915s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.434055492Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.404524113s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.442799398Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 736.398884ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.488860835Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.48533082s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.703624702Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 566.529213ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.770925117Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.06347513s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.83583857Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.031594869s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:32.938215769Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 743.61351ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.066831145Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.370092995s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.07866074Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 549.095762ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.087732904Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.239437963s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.09532383Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.169866588s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.131520733Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 1.00081203s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.234085027Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 1.178649652s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.311617135Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 990.116657ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.336072626Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.030929336s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.345199231Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.385411012s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.353766442Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 1.159157549s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.647245688Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 505.992332ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.692409742Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 880.988277ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.747078052Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.166156111s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.782434423Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 1.375840107s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.81841074Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.343464996s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.85522194Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 609.66024ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.907334823Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 804.098697ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:33.981037623Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.17697749s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.199038483Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 538.241058ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.207690656Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 805.385776ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.244495325Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.264227992s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.311631285Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 1.439527894s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.335258624Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 503.871562ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.344759159Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:34.350685202Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:34.35099146Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.470387874s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.385868809Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.336339529s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.458928412Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 874.733056ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.499863658Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 677.760379ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.527815828Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 27.839636ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.544760845Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 90.917891ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.579538962Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 794.931919ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.636783838Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 261.969097ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.680441093Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 920.623484ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.761606106Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 1.261027865s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.774617381Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.20739069s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.797503121Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 511.781675ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.817147847Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.314283516s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.861955273Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 478.932864ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.958180653Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 973.535185ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:34.971445775Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 878.971495ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.063277112Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 1.284220767s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.208388213Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 908.848284ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.220083485Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 697.126595ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.234069366Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 1.362479981s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.349966073Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 1.12307933s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.390471213Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.020610413s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.403625546Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 585.018641ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.445553427Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 827.576478ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.456714285Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.108211156s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.588702418Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.338802093s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.694278579Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 1.467526467s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.790659224Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.310403076s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.808700499Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.220619847s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.869876506Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.411574173s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.946276916Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.080258343s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.961448967Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 838.744811ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:35.976182497Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 937.103834ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.021391912Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 1.210544297s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.04729882Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 696.614487ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.062988045Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 999.638632ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.143132401Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.084971093s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.190297827Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 1.181679566s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.298820573Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.455451426s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.436885173Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 519.502256ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.504877546Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 1.194326929s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.525656151Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.417379043s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.712343364Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 913.261214ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.726835788Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 545.593872ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.77543594Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 631.237979ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.829628307Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.335054872s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:36.973024784Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 796.322148ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.038596794Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 533.170033ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.051893577Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 873.185484ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.064373809Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.360043229s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.106382722Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 965.292466ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.131806957Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 741.760501ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.158917527Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 904.464274ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.274725681Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 743.601073ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.288988979Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 810.995835ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.303637847Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 983.865521ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.337668044Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 660.996908ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.375578852Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 682.033852ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.47712893Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 643.49733ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.505875311Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:37.532328482Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:37.532809755Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.243509294s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.598864738Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 1.351416106s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.656277258Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 1.252783169s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.742449584Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 27.131338ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.844445509Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 1.422241239s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.854872938Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 116.891327ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.883316881Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 551.059512ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.90463064Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.480437607s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.916844365Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.435725864s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:37.953941102Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 179.994405ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.018520446Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 984.879511ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.055019287Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 994.332984ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.073787521Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.029765608s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.116698547Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.149796207s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.135908083Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.144894539s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.155944139Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 552.758766ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.174050346Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 520.87659ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.194869443Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 1.293601391s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.212926944Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 271.577287ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.228820318Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.329552056s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.321046785Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 512.194378ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.333297091Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.078767688s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.468514087Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 969.32387ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.478033382Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 1.049705081s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.512599433Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 598.022107ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.730402118Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 949.351103ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.74443012Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.161895154s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.857794603Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 1.01680392s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:38.873567029Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 720.144364ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.016564765Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 513.623141ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.037502805Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 1.42705519s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.06213946Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 841.724376ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.082529866Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.465657694s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.165528156Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 1.466622492s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.292780076Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.126415421s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.305905525Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.476805462s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.319140386Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 525.949612ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.338071914Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.263970981s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.401289484Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 765.604445ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.411839436Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 677.72159ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.457560744Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 802.251409ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.517702785Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 1.391567126s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.558738013Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 1.484532679s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.568070853Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 1.130338547s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.577829305Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 896.940518ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.662119758Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 674.387991ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.708773521Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 1.118040864s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.749371627Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 1.22311495s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.875320084Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 1.341976688s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.900641187Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.295888251s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:39.934005767Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.03120434s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.035309843Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 735.011473ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.147966903Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.432989396s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.191170667Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 1.477687913s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.333858905Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 1.472884131s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.396033527Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 579.480757ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.446378235Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 932.213969ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.503123868Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 632.798876ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.575098824Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.013108315s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.643579739Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 1.490550498s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.678349496Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 1.143982146s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.721650632Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 847.566725ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.771102937Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 715.27336ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.811478978Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 936.950152ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.880097546Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 553.618434ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.898885821Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 978.675729ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:40.962292256Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 843.540709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.044018872Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe\") failed: Property not found. retry in 1.496648491s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.064054559Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 791.181069ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.077985198Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 676.863126ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.109426859Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 741.386924ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.224624528Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.321918015s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.247790406Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 954.585095ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.259269941Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 573.896572ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.451480196Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954\") failed: Property not found. retry in 1.211222567s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.466997596Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 626.027154ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.539421054Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 765.725753ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.631817365Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a\") failed: Property not found. retry in 1.110069527s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.645269672Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058\") failed: Property not found. retry in 1.110086774s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.669486797Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938\") failed: Property not found. retry in 1.105992329s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.803652506Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4\") failed: Property not found. retry in 1.400011269s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.826268069Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5\") failed: Property not found. retry in 901.266724ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.839758906Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301\") failed: Property not found. retry in 846.012668ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.879164362Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843\") failed: Property not found. retry in 893.292601ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.891613133Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69\") failed: Property not found. retry in 1.199877058s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.902214386Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a\") failed: Property not found. retry in 1.14388933s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.916336639Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914\") failed: Property not found. retry in 901.091713ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.958153729Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25\") failed: Property not found. retry in 1.169123369s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:41.979615902Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079\") failed: Property not found. retry in 598.825919ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:42.036799286Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\") failed: Property not found. retry in 1.412712973s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:42.176092997Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846\") failed: Property not found. retry in 828.729074ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:42.184510304Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65\") failed: Property not found. retry in 879.831934ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:42.227001349Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd\") failed: Property not found. retry in 889.080203ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:42.410225673Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143\") failed: Property not found. retry in 1.05388824s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:42.585592964Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:42.619925429Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:42.61997761Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:43.185491009Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:43.185627632Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:46.374654714Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:46.374722475Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::a exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:51.93820541Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::a, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:51.938268332Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.486806373Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.452602982s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::a exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:52.491173115Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:52.491187775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.493775802Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.452830017s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.495219919Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.460211497s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.495548508Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.462114627s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.495810525Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.460782632s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.806642528Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.771533004s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.815860196Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.782623199s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.816279346Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"21.321141887s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.817041576Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"216.318705509s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.826095919Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.792393211s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.826236953Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.794308771s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.826324935Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.792987766s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.827223969Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.792613167s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.82729061Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.793769486s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.82768214Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.794452984s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.827934097Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.792235627s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.828121452Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.792676488s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.828305036Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.793030767s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.828524742Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.794729142s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.828576553Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.797308798s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.828643615Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.793061619s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:52.82882703Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.797088362s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:52.944504632Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:52.944550623Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:54.184316504Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:56.079977383Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"230.044577723s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:20:56.080091876Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"230.045662651s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::c exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:59.459163517Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:20:59.459229039Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::c exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:00.016611118Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:00.01668706Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.021423592Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"233.985571192s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.021589866Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"233.987292615s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.021707109Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"233.989644287s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.021815458Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"233.990417096s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.021911255Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"233.986882745s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.046122049Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"234.01422263s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.066880914Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"234.033216459s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.245998451Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"234.212180243s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:00.376107975Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"234.340940372s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:00.567902169Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:00.5679203Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:01.14898393Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.11352696s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:01.285981921Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.25239859s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:01.345961178Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.312486268s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:01.514339808Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.478741215s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:01.514381309Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.479863644s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:01.725982978Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.690676558s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:01.756748908Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62892"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:01.757918628Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"bddd3036-cc4b-4f61-b3e9-16773acdb76e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62892","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":658,"response_code":"200"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:02.096042205Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"225.597924638s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:02.806036179Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"236.772822566s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:02.945996777Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"236.91026979s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:02.955952404Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"236.922014653s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:03.26613347Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"237.231239644s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:03.316120098Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"237.281201663s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:03.326018674Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"237.291921749s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:03.78280921Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"237.750930592s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:03.936100982Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:03.938587156Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"227.44053537s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:03.985766412Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"237.952119048s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:04.05510149Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"238.023712145s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:04.086012337Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"238.052676471s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:04.442307641Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"238.410246438s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:04.938069402Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"238.903063333s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:04.953589132Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"238.917791353s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:05.345078415Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"239.309952962s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:05.641190009Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"239.606930059s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:05.867244287Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:05.869812123Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"229.371794057s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:06.197692606Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"240.163915598s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:06.358947383Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"240.323559393s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:06.423160588Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"240.388714703s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:06.584068627Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"240.550912464s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621088701Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_de28215d-5e75-4ba0-aaaf-7fc4933ba4c2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621156623Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_6ebf9979-ce51-4c91-a9a2-011bb6255058"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621199924Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_external_dns_de116760-7ce8-4051-89b5-dbe725a06843"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621209774Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_b05778b9-9d7a-46ab-9333-eb3104c321fe"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621219335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_2063053f-6bdd-4615-8127-179923ae9a65"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621229775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_bc500983-cf20-4c4d-8f8c-f5db945b3e2a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621238905Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_49abcdd7-0bae-4ae0-88f1-0467b03df954"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621253535Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_a7894346-dd44-48fa-aeb3-3d055054a846"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621263056Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_external_dns_0ed061a1-d644-4f36-95d9-99c559521914"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621271996Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_8dbbe7bd-5bb7-4059-a33a-2e6c08694fbd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621282546Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_c263e526-2219-421f-a41f-871118e2593a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621293206Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_oximeter_406ab4c7-818a-4298-ae6e-64e3dfad5143"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621311797Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_a839add2-c178-4fc2-ac03-7efdcd3ad938"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621320707Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_ca7c0b15-dbe1-4587-87c3-9086e4f965d4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621327697Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_6e567479-af48-4e4b-925c-59aa4134ba69"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621337187Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_b8c00cc1-cb6b-4f24-b10d-eb3f06dedf25"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621350508Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_c803c23d-6aa1-46ff-a7bf-b16818f7abe5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621369918Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_c36baa1a-e79a-4644-8a6b-9dc975d1f079"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.621388369Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_clickhouse_00b7bf82-35cc-48fe-9476-270d45e6d301"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.62219829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.630022751Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.6509042Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"eb1a1298-03ff-4c18-80a1-d84f63c3e215","remote_addr":"[fd00:1122:3344:101::1]:51895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":55472690,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.663868204Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.664001307Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.664198303Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.667228171Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.667234281Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.707399796Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/9ebadecf-c131-4403-83ea-910555047c33/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.709368837Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:06.793816254Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"240.758567698s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/2c331470-b440-4c33-9aa1-35acb298a49d/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/9ebadecf-c131-4403-83ea-910555047c33/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.80179725Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/9ebadecf-c131-4403-83ea-910555047c33/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:06.92205412Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/9ebadecf-c131-4403-83ea-910555047c33/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:07.056431824Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 403 Forbidden; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4e1516de-a049-4eb9-80dc-65351fec7fea\", \"content-length\": \"113\", \"date\": \"Wed, 28 Feb 2024 13:21:07 GMT\"}; value: Error { error_code: Some(\"Forbidden\"), message: \"Forbidden\", request_id: \"4e1516de-a049-4eb9-80dc-65351fec7fea\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:07.273804728Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4d60dd5-1cae-4f8e-9c55-c6919b9b3308\", \"content-length\": \"160\", \"date\": \"Wed, 28 Feb 2024 13:21:07 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: ip-pool with name \\\"oxide-service-pool\\\"\", request_id: \"d4d60dd5-1cae-4f8e-9c55-c6919b9b3308\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:07.487529808Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:07.64778716Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"241.61437976s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ab05d276-1ab6-4f99-8799-af8711b5a769\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"2c331470-b440-4c33-9aa1-35acb298a49d\\\\\\\"\\\", request_id: \\\"ab05d276-1ab6-4f99-8799-af8711b5a769\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"720a18d5-3c11-45d6-bab4-6d04bdd17f10\", \"content-length\": \"160\", \"date\": \"Wed, 28 Feb 2024 13:21:07 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: ip-pool with name \\\"oxide-service-pool\\\"\", request_id: \"720a18d5-3c11-45d6-bab4-6d04bdd17f10\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:07.711983225Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:07.865481812Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"241.831611541s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"32ba8fe9-509e-468d-90dc-51b213b9ba44\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"2c331470-b440-4c33-9aa1-35acb298a49d\\\\\\\"\\\", request_id: \\\"32ba8fe9-509e-468d-90dc-51b213b9ba44\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:07.912618907Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"241.881280712s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"47db3408-ec07-4fe2-a632-8b1955134c21\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"2c331470-b440-4c33-9aa1-35acb298a49d\\\\\\\"\\\", request_id: \\\"47db3408-ec07-4fe2-a632-8b1955134c21\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:07.961304752Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"241.925799429s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"44a2bbe7-3066-4086-a5e9-eed9459e0035\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"44a2bbe7-3066-4086-a5e9-eed9459e0035\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"98771a62-165f-42fd-b7dc-481512522747\", \"content-length\": \"160\", \"date\": \"Wed, 28 Feb 2024 13:21:07 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: ip-pool with name \\\"oxide-service-pool\\\"\", request_id: \"98771a62-165f-42fd-b7dc-481512522747\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:08.146942958Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:08.206025141Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"242.172357975s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"087a98a7-8d30-445f-8263-00d79368dea0\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"2c331470-b440-4c33-9aa1-35acb298a49d\\\\\\\"\\\", request_id: \\\"087a98a7-8d30-445f-8263-00d79368dea0\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:08.31340519Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"242.277767023s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"3366551f-93c8-40e6-9519-0950f6087ea6\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:08 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"3366551f-93c8-40e6-9519-0950f6087ea6\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"69f7d1a9-6718-444c-835d-2ce6a9f3ccb1\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:21:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"69f7d1a9-6718-444c-835d-2ce6a9f3ccb1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:08.630219887Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d6c3efa6-ecfc-450a-9a0c-af787b8404f8\", \"content-length\": \"124\", \"date\": \"Wed, 28 Feb 2024 13:21:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d6c3efa6-ecfc-450a-9a0c-af787b8404f8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:09.149045803Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:09.365154394Z","hostname":"buskin","pid":1671,"sled_id":"2c331470-b440-4c33-9aa1-35acb298a49d","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:09.398312808Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.365215466s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"8cbcaf7c-7b92-4f99-a7e9-163267cb4952\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:09 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"2c331470-b440-4c33-9aa1-35acb298a49d\\\\\\\"\\\", request_id: \\\"8cbcaf7c-7b92-4f99-a7e9-163267cb4952\\\" }\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:09.42903158Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"2c331470-b440-4c33-9aa1-35acb298a49d"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:10.196985419Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"244.162349247s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"91348993-ebe7-4318-8212-47e77d17f171\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:09 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"91348993-ebe7-4318-8212-47e77d17f171\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b6e24c3f-3b01-4402-9644-ede65dc45ecb\", \"content-length\": \"174\", \"date\": \"Wed, 28 Feb 2024 13:21:09 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"b6e24c3f-3b01-4402-9644-ede65dc45ecb\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:10.23854381Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:10.939012638Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"244.903692979s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"3fa85638-0dde-45e7-b4d1-da880844890c\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:10 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"3fa85638-0dde-45e7-b4d1-da880844890c\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:10.9635107Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"244.928482487s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"78241c48-7261-45db-9e11-ebcc46e93dce\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:10 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"78241c48-7261-45db-9e11-ebcc46e93dce\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"98128f65-963a-4e03-8a61-fe95fdddc86b\", \"content-length\": \"174\", \"date\": \"Wed, 28 Feb 2024 13:21:10 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"98128f65-963a-4e03-8a61-fe95fdddc86b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:11.24940113Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:11.389735478Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.354962532s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"c5d3a494-a070-4f7a-aed1-68d793d03272\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:10 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"c5d3a494-a070-4f7a-aed1-68d793d03272\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:12.179414406Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"246.14384133s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"a5a7d96b-8d20-4d9f-96e8-5317f1c234a5\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:12 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"a5a7d96b-8d20-4d9f-96e8-5317f1c234a5\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b722937b-9411-4ad6-8ddc-c4d29fc2a895\", \"content-length\": \"174\", \"date\": \"Wed, 28 Feb 2024 13:21:12 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"b722937b-9411-4ad6-8ddc-c4d29fc2a895\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:12.27345606Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:12.436709819Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"246.402558859s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9d232326-2972-4d2f-a172-c0bb9f695d1f\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:12 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"9d232326-2972-4d2f-a172-c0bb9f695d1f\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bc2d84ed-0c04-49cd-96e2-039cfc3980a4\", \"content-length\": \"174\", \"date\": \"Wed, 28 Feb 2024 13:21:13 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"bc2d84ed-0c04-49cd-96e2-039cfc3980a4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:13.69697632Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:14.415951685Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:14.415976276Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:14.415982036Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:14.705438589Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"248.670326972s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"b27c07d1-2972-4ed8-8111-be5df0a8909e\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:14 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"b27c07d1-2972-4ed8-8111-be5df0a8909e\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c7302aea-3a3a-43ac-97c2-6b0375321b1d\", \"content-length\": \"174\", \"date\": \"Wed, 28 Feb 2024 13:21:14 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"c7302aea-3a3a-43ac-97c2-6b0375321b1d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:15.48128094Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:21:15.50687239Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"249.472168414s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ce87cea4-03ed-4b8e-9669-9bb220077ac5\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 28 Feb 2024 13:21:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"ce87cea4-03ed-4b8e-9669-9bb220077ac5\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26b27863-7437-4a93-b133-aef5ca73d848\", \"content-length\": \"174\", \"date\": \"Wed, 28 Feb 2024 13:21:18 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"26b27863-7437-4a93-b133-aef5ca73d848\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:18.24965584Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081401413Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081429453Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081436373Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081442004Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081455754Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081461354Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081466854Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 2c331470-b440-4c33-9aa1-35acb298a49d, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.081473005Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f5120d0-8ca8-4507-87f1-1baaa29b1631\", \"content-length\": \"174\", \"date\": \"Wed, 28 Feb 2024 13:21:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"0f5120d0-8ca8-4507-87f1-1baaa29b1631\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:19.67283912Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:21.161312284Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:21.161338044Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:21.161470368Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:21.161478898Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:21.161498549Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.553717801Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46935"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563541724Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563594526Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563614136Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563621956Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563629146Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563635657Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563641767Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:23.563660157Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ba74797b-672e-4a06-8527-51d4c672b104","remote_addr":"[fd00:1122:3344:101::c]:46935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9654,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"501d37b2-298d-4465-a175-1921f5e32a0e\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:23 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"501d37b2-298d-4465-a175-1921f5e32a0e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:24.054711607Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:24.132208575Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:62926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:24.133735404Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"0e51dbfa-9b81-4ae6-848f-3c259df576fe","remote_addr":"[fd00:1122:3344:101::c]:62926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1450,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:24.134713109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:24.135050978Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:24.136858395Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:24.13706725Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"05cd9177-c624-4f68-8445-f2a0b53be1af","remote_addr":"[fd00:1122:3344:101::c]:62926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3127,"response_code":"200"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:25.225137191Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:25.225160222Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:25.225166172Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:25.225170522Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:25.225174932Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:25.225179292Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.473756541Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37253"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.474936461Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.474972712Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.474998783Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.475019383Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.475031684Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.475039384Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.475058454Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.475065275Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8ec65615-c534-458c-91f0-1df71fcd0f71","remote_addr":"[fd00:1122:3344:101::c]:37253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1056,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b811fb5-d588-4e56-90a4-e1327c7be641\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:26 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9b811fb5-d588-4e56-90a4-e1327c7be641\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.551135116Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.586780004Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.58738181Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"a1f9e89c-4cba-4be5-8bad-de4590017c75","remote_addr":"[fd00:1122:3344:101::c]:56006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":584,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.587547334Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.587567235Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.587964925Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:26.588676193Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"3c4d6369-20f9-4f29-83c0-4b8212dc2dcd","remote_addr":"[fd00:1122:3344:101::c]:56006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.874525544Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:41587"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875588442Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875637383Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875668104Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875686434Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875694694Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875714325Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875721415Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.875728235Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0bcf6724-1404-44e1-8d35-e5bfd17af5bf","remote_addr":"[fd00:1122:3344:101::c]:41587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":937,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9785ae69-4a74-453d-9889-f65e20b774fb\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:29 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9785ae69-4a74-453d-9889-f65e20b774fb\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:29.981252736Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:30.144854184Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:54288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:30.144922565Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"336ce8fb-fbc9-47bb-8a62-713109b79810","remote_addr":"[fd00:1122:3344:101::c]:54288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:30.14510234Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:30.145121751Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:30.145318145Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:30.145514761Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"19505b46-d22b-472f-ae15-5ccc2e8bc645","remote_addr":"[fd00:1122:3344:101::c]:54288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":433,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:31.794447331Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:31.794533893Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"dfef02a8-10cd-49f0-896f-53c3796a688e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57400","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.622764155Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36098"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.62294205Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.622997471Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.623028762Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.623039432Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.623048252Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.623057102Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.623065803Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.623086403Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ac9612b0-14a2-4e7a-80ad-5f6eed7dc9cf","remote_addr":"[fd00:1122:3344:101::c]:36098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":302,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3bb4db0b-5171-424f-a717-cd71d93e757d\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:32 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"3bb4db0b-5171-424f-a717-cd71d93e757d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.696377773Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.891481413Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:50011"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.891624546Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"7ddd9a92-66da-4a19-a064-f6178f5f6c75","remote_addr":"[fd00:1122:3344:101::c]:50011","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.891907254Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.891937194Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.892131559Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:32.892312664Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"3af0cac0-2889-47a3-829b-50a3e3ef1445","remote_addr":"[fd00:1122:3344:101::c]:50011","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":451,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149520776Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:65031"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149760493Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149816254Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149851895Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149861215Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149869405Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149881716Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149893606Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.149934007Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d2e32f79-9f6a-4c5e-8cea-08d17fe23011","remote_addr":"[fd00:1122:3344:101::c]:65031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":351,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ff7aec33-d824-48f0-868c-0c05288d981a\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:36 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ff7aec33-d824-48f0-868c-0c05288d981a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.208777054Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.334494355Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:44682"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.335164612Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"448fd106-d937-4750-9655-ee6390390b2b","remote_addr":"[fd00:1122:3344:101::c]:44682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":656,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.335366378Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.335394148Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.335605544Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:36.335808179Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"b3d6b423-4cad-4671-b04e-bad5adc1fdbd","remote_addr":"[fd00:1122:3344:101::c]:44682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":470,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.966929121Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:48042"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.967153617Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.967199428Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.967240779Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.967249499Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.967255919Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.96726707Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.96727809Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:41.967316461Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fe45001d-05e0-49b6-a054-74de084ff317","remote_addr":"[fd00:1122:3344:101::c]:48042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":328,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca1000ef-b326-4b21-871a-0ae7b53ecffd\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:42 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ca1000ef-b326-4b21-871a-0ae7b53ecffd\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:42.028276523Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:42.164622178Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:40652"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:42.164697289Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"211d1921-9800-4123-896e-e2d16654a83c","remote_addr":"[fd00:1122:3344:101::c]:40652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":50,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:42.16547312Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:42.16549986Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:42.165692895Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:42.16586075Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"12adabbd-dce1-474f-9642-ca74dd193341","remote_addr":"[fd00:1122:3344:101::c]:40652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1024,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:44.253047918Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:38246"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:44.257459342Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"53b98b88-3b5e-4c8f-979f-e910bd284d3d","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":4378,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.449759841Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63147"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450125271Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450156782Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450189952Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450197503Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450204493Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450209953Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450215573Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.450228643Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a032f193-a118-479d-a62a-be67c9d9b41d","remote_addr":"[fd00:1122:3344:101::c]:63147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":228,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ffbd7981-a73a-4b4d-9fc2-11f0c474a297\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:48 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ffbd7981-a73a-4b4d-9fc2-11f0c474a297\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.514801838Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.716884228Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.71696666Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"35f4dab5-d3de-4ac3-96d3-3a96b20916dd","remote_addr":"[fd00:1122:3344:101::c]:46971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.717154405Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.717186456Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.717406491Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:48.717616567Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"b37accdc-49c5-4f1a-9d6d-507ef115770f","remote_addr":"[fd00:1122:3344:101::c]:46971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":485,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.415973954Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59203"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.416142438Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.41617813Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.41619943Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.41620728Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.41621349Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.41622018Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.416231091Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.416249011Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b94c2b34-61a7-431c-a979-8055462ffcc4","remote_addr":"[fd00:1122:3344:101::c]:59203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":240,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e67bba7a-26f1-4db6-83c1-b8d54b1d44b5\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:21:57 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"e67bba7a-26f1-4db6-83c1-b8d54b1d44b5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.490972258Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.673858173Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:44203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.673907024Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"9bd1567a-c5fb-4019-b8c8-3e79dea1f6b5","remote_addr":"[fd00:1122:3344:101::c]:44203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.674063978Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.674083108Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.674302784Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:21:57.674495909Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"d1f3b601-1074-4f9d-a7ce-a69796b12eb0","remote_addr":"[fd00:1122:3344:101::c]:44203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":460,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:01.826264804Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:01.826335945Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"66386ef5-bd1e-453e-9ea2-30c23363bda4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59109","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017196977Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51443"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017420032Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017484984Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017506895Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017533285Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017543856Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017554226Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017564046Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.017574506Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6f13cb29-d067-40a5-a4cb-e0596c507e0b","remote_addr":"[fd00:1122:3344:101::c]:51443","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":338,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26466c69-be48-46fd-ac1e-2de8000a8afc\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:22:04 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"26466c69-be48-46fd-ac1e-2de8000a8afc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.065925373Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.192741612Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42641"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.192830444Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"8087912b-388e-499d-b0ab-3e1ab03aaf9e","remote_addr":"[fd00:1122:3344:101::c]:42641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":59,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.19304754Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.193073511Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.193344788Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:04.193565603Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"5e202d26-a2ed-4609-8542-27853f4b6c72","remote_addr":"[fd00:1122:3344:101::c]:42641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":554,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527161146Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34731"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527351371Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527389292Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527414153Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527425783Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527435593Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527445644Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527456484Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.527466234Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"dd31dcaa-fde2-4eea-8d40-55cabdc1884b","remote_addr":"[fd00:1122:3344:101::c]:34731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":242,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"839c3f4a-0ed5-48a7-b0ce-5ecfa3a2ff76\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:22:13 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"839c3f4a-0ed5-48a7-b0ce-5ecfa3a2ff76\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.589833632Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.723228431Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:38581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.723287113Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"300b4602-bbca-49a9-a32a-5a059eec1ba2","remote_addr":"[fd00:1122:3344:101::c]:38581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.723436386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.723457777Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.723682453Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:13.723857397Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"07e48ef7-626e-4c7d-89bf-b8e022780d2a","remote_addr":"[fd00:1122:3344:101::c]:38581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":444,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:14.252706531Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"00fb8f42-3917-4456-ba05-53a095b47dd9","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":195,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.312832654Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:64949"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.312982438Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.313020248Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.313033769Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.313044579Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.313053739Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.31306301Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.31307262Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.31308223Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"322ad13c-878d-4d3f-aea6-0d8bb2353aee","remote_addr":"[fd00:1122:3344:101::c]:64949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":226,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"857a0362-8b0c-4da4-9ca6-083a06c9e141\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:22:23 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"857a0362-8b0c-4da4-9ca6-083a06c9e141\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.369176836Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.461509037Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37848"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.461570058Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"21edb5a7-ac64-4fc1-b4da-10b130063c16","remote_addr":"[fd00:1122:3344:101::c]:37848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.461755753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.461777243Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.46201379Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:24.462172514Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"60d3d09c-790e-4011-a16a-cf7c783890d8","remote_addr":"[fd00:1122:3344:101::c]:37848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":443,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:31.859986601Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55082"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:31.860648999Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"c4eec44c-6d6c-4db6-a174-e44819438070","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55082","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":644,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.219411559Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49100"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.219656536Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.219721887Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.219764639Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.219772639Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.219782589Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.219793949Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.21981254Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.21984045Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"751fc263-6c0e-473b-b089-ab2244869a87","remote_addr":"[fd00:1122:3344:101::c]:49100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":355,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b487dfc1-8107-4eed-b7ca-8ff2587e21d2\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:22:43 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b487dfc1-8107-4eed-b7ca-8ff2587e21d2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.271158163Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.361277956Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34228"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.361337608Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"3b76d448-ad49-4df4-89ed-5d441d016590","remote_addr":"[fd00:1122:3344:101::c]:34228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.361477261Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.361507542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.36181668Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:43.362076067Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"6710f294-9a77-4723-a45a-2c045334f280","remote_addr":"[fd00:1122:3344:101::c]:34228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":622,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:44.252234136Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"d2f3db17-22fe-4c96-a877-705abb14cb1b","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":264,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:48.128365163Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.223660083Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51396"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.223857898Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.22391225Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.22392812Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.22393855Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.223971461Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.223981231Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.223989992Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.223998552Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c75b5d49-899c-40e0-9ecd-92563ccae1a5","remote_addr":"[fd00:1122:3344:101::c]:51396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":285,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df1189fb-598b-404c-83fe-02658a6633c1\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:22:59 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"df1189fb-598b-404c-83fe-02658a6633c1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.278034394Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.374546253Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63147"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.374630565Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"5595a860-5af6-4a73-8259-10df959495a2","remote_addr":"[fd00:1122:3344:101::c]:63147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.375380064Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.375414725Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.37562775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:22:59.375794455Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"9300dc7a-89d1-4430-b340-62f76736bc28","remote_addr":"[fd00:1122:3344:101::c]:63147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":984,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:01.900166544Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:01.900217765Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5bef1cf3-a6e7-4f32-acaa-c33d1be1242e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41545","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:14.252930561Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"1f64d67c-04b3-4952-9cc1-c1e4fd27905f","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":532,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872069263Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:53551"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872220047Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872269098Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872287149Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872293629Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872299049Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872307269Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.872313269Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.87233871Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0bf8f413-e098-409c-b638-cde9ddfd5c63","remote_addr":"[fd00:1122:3344:101::c]:53551","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":230,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"31dbc2d0-c1c5-411d-93e3-b2b0b860c77c\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:23:25 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"31dbc2d0-c1c5-411d-93e3-b2b0b860c77c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:25.92239201Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:26.011959919Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:64622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:26.012023791Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"980fd835-2b10-4bb2-bdf5-a36c06bdcffc","remote_addr":"[fd00:1122:3344:101::c]:64622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:26.012861653Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:26.012879203Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:26.01313852Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:26.013324154Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"a29017c5-c3f4-4a1a-847f-e05d6a3558e9","remote_addr":"[fd00:1122:3344:101::c]:64622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1183,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:31.931786593Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57510"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:31.931840995Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5f499ecc-e914-4712-8864-8189baa94646","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57510","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:44.252360959Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"5e64732c-c1b8-4c75-a5bd-03026d18e64e","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":290,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.944565725Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36844"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.944706648Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.94474678Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.94477514Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.94478256Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.944788221Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.944793751Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.944802681Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.944815251Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fece8c67-7c5e-413f-91ce-d8d15038e9f9","remote_addr":"[fd00:1122:3344:101::c]:36844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":229,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b12c09e8-f2d1-43ef-9497-05e68923de8c\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:23:45 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b12c09e8-f2d1-43ef-9497-05e68923de8c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:45.993804024Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:46.085316983Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:46.085383005Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"f2d53f41-0098-420b-8815-95615e43dfb1","remote_addr":"[fd00:1122:3344:101::c]:51838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:46.085546889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:46.08556598Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:46.085834577Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:23:46.086034232Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"6881d0e3-b971-43c9-809e-a448645711c7","remote_addr":"[fd00:1122:3344:101::c]:51838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":510,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:01.964208412Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43886"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:01.964257223Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"b23974bb-d1cd-46ea-9e75-2cb5382f3298","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43886","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:14.252122043Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"c9a50b0f-cae6-4fa7-8b9e-dade584d087b","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":610,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.11906813Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34461"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121081922Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121111663Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121132124Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121139344Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121153664Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121165134Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121171385Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.121200765Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0c00a8bf-12a5-44c2-81b6-aff4cb5bd938","remote_addr":"[fd00:1122:3344:101::c]:34461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1918,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec4671bf-47b7-4473-860e-39e07f9a5f1e\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:24:27 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ec4671bf-47b7-4473-860e-39e07f9a5f1e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.180873694Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.283636973Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.283704165Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"4e5d5863-7080-4a5a-86c1-2b328ccf7027","remote_addr":"[fd00:1122:3344:101::c]:59381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.284467004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.284487385Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.284751682Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:27.284948287Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"053a1f7d-3ef2-4ff6-b1a6-018a943141dd","remote_addr":"[fd00:1122:3344:101::c]:59381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1099,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:31.998963763Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48039"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:31.999010165Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"1d33118b-0b1d-45d3-86f1-3b34361d9fab","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48039","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:44.252003905Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"c19bc6c6-a1a7-4bdb-8bc6-98aad8c64dc1","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":290,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.148436124Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:40773"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149163052Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149201884Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149216884Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149223244Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149229404Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149236364Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149242494Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.149262125Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9c06b176-03ae-4a10-afa8-8f9c24c3647f","remote_addr":"[fd00:1122:3344:101::c]:40773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":793,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"18a23092-49f5-491b-80e6-a69909b99b21\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:24:58 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"18a23092-49f5-491b-80e6-a69909b99b21\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.203824352Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.302013753Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:53948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.302070914Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"92ab87d4-119f-48fd-9a29-c0792f725985","remote_addr":"[fd00:1122:3344:101::c]:53948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.30227375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.3023023Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.302563037Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:24:58.302779403Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"9cb8ae2d-aaae-498e-aab4-94ecdfe5282e","remote_addr":"[fd00:1122:3344:101::c]:53948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":523,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:02.032166416Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57677"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:02.032234128Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"26985371-f8d1-420c-8269-5e017d267946","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57677","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:14.253039149Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"9184cbb2-fc7a-46c3-8fa6-9229c7fb1fed","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":326,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:32.064283383Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:32.064329984Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"8c8dd048-7adc-4937-a2e2-cf1682b36e49","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47397","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.24628551Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42768"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.246924277Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.246970798Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.246992049Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.247000009Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.247007399Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.247013879Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.247020549Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.24705134Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"20f30714-c238-4264-9415-a62e0176632a","remote_addr":"[fd00:1122:3344:101::c]:42768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":732,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"15b1592b-398f-400b-b7c0-b81e83ca9e2f\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:25:39 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"15b1592b-398f-400b-b7c0-b81e83ca9e2f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.302878749Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.398273658Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36156"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.398323619Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"875a9b0a-3579-4831-89fb-577b8e4e1a45","remote_addr":"[fd00:1122:3344:101::c]:36156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.398498034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.398526095Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.398811952Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:39.399027177Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"367e491a-a946-4acf-88a8-35ebd8f85255","remote_addr":"[fd00:1122:3344:101::c]:36156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":544,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:25:44.252250193Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"6752f042-7006-4e4a-bf6f-b1cb8d016de4","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:02.096979179Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54719"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:02.097035181Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"71cdd2f9-3367-404e-8e08-00041805a6aa","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54719","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573219046Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52797"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.57337667Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573435082Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573447612Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573458082Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573466462Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573475483Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573489973Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.573499563Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"211e8aae-6de6-4a41-9fc4-df8a71f55c07","remote_addr":"[fd00:1122:3344:101::c]:52797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":256,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e693dd6f-d2cb-4062-a302-4d5222812769\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:26:07 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"e693dd6f-d2cb-4062-a302-4d5222812769\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.628702106Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.731617629Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:35780"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.731684101Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"ecfff175-b34f-4978-a67c-27243b2855f8","remote_addr":"[fd00:1122:3344:101::c]:35780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.731830045Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.731854526Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.732071761Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:07.732232385Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"b78a7869-cb16-4b21-8dec-df5797d11e66","remote_addr":"[fd00:1122:3344:101::c]:35780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":425,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:14.252525488Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"fcf2e001-2d86-48b9-aa72-bb40e8474fee","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":284,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:32.13048843Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:32.130564892Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"0b71673d-6087-4d6c-840e-5f517b7ee518","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51192","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:26:44.252636969Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"d02707de-371f-480f-9757-e8436ef1efaf","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":293,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:02.162048853Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37246"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:02.162114184Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"e21d60c7-5797-47fc-bda3-4de1b9b79f7b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37246","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:03.540286193Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:03.540325724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:03.576079426Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.99999536s","next_cleanup":"Instant { tv_sec: 18118, tv_nsec: 162007225 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.79811627Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49270"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799095445Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799144936Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799171317Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799181477Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799193438Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799209708Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799226178Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.799240769Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"22cd5d35-aa9f-488a-9e4d-df2997ef43a4","remote_addr":"[fd00:1122:3344:101::c]:49270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1036,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2b52ded4-9d1a-43ff-bff5-04760ddc7402\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:27:12 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2b52ded4-9d1a-43ff-bff5-04760ddc7402\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.845006268Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.936086976Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:53348"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.936726293Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"2fe9b00a-758b-41aa-a3b3-4574bd2c3a38","remote_addr":"[fd00:1122:3344:101::c]:53348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":618,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.936908658Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.936944949Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.937212116Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:12.937451322Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"b8cbe62b-83ab-47ab-af69-15064654430f","remote_addr":"[fd00:1122:3344:101::c]:53348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":601,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:14.252549875Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"ffe1c9fc-f035-4b3d-9b1c-af645f9bdc05","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":433,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:32.194393054Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44969"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:32.194500947Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"4f2a4eaa-b8f0-48b3-8490-0194b81b500e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44969","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.737546306Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40919"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.748790106Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:586","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-28T13:27:40.433945Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(931998))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-14aa8e10166\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 230cb32f-7c99-48aa-bfc0-b23e19f98184, read_only_parent: Some(Volume { block_size: 512, id: 0c3be992-a0a3-45c4-b106-087fb22ccd4c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ed895732-7459-4e8d-baac-5393d9418000, key: Some(\"14qtOIxuHCKGHRjgQ2G4QxGZjDCCs5BPzi0wqTKoCtQ=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 230cb32f-7c99-48aa-bfc0-b23e19f98184, key: Some(\"dchOU53ju6AuCY+zZ6G8pD2HufppaHs+QlUJ6aIyAVM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABGdGtcWFxYAAB0a1xYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAARnRrXFhcWAAAdGtcWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiMTMxNzQ3Zi0zYTlkLTQ4ZDgtOWM5Mi04Zjc2ODEyOGFkY2MiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSnI4ZHBKaFVHekxpTEo1ZTBXd1M5SjExbzZSRzlRN1ZrTnNLc0EzMVRGbSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.74894174Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:619","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.749397541Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance.rs:932","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(931998))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: cb015a45-bd55-49a7-878e-451e1a99bc53, kind: Instance { id: b131747f-3a9d-48d8-9c92-8f768128adcc }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 25, 160])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(931998), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-14aa8e10166\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 230cb32f-7c99-48aa-bfc0-b23e19f98184, read_only_parent: Some(Volume { block_size: 512, id: 0c3be992-a0a3-45c4-b106-087fb22ccd4c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ed895732-7459-4e8d-baac-5393d9418000, key: Some(\"14qtOIxuHCKGHRjgQ2G4QxGZjDCCs5BPzi0wqTKoCtQ=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 230cb32f-7c99-48aa-bfc0-b23e19f98184, key: Some(\"dchOU53ju6AuCY+zZ6G8pD2HufppaHs+QlUJ6aIyAVM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABGdGtcWFxYAAB0a1xYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAARnRrXFhcWAAAdGtcWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiMTMxNzQ3Zi0zYTlkLTQ4ZDgtOWM5Mi04Zjc2ODEyOGFkY2MiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSnI4ZHBKaFVHekxpTEo1ZTBXd1M5SjExbzZSRzlRN1ZrTnNLc0EzMVRGbSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-28T13:27:40.433945Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.74972197Z","hostname":"buskin","pid":1671,"uri":"/instances/b131747f-3a9d-48d8-9c92-8f768128adcc","method":"PUT","req_id":"52da6f5e-ea67-49e1-97a2-d2ca6e532527","remote_addr":"[fd00:1122:3344:101::a]:40919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":12099,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.834765672Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53249"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.855322552Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 242, 25, 160]), slot: 0, vni: Vni { inner: 931998 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.856188205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.890070688Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:40.910234148Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:42.816807429Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:42.817006864Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 70.27499ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:42.995247179Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 90.396778ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:43.083044283Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 128.642238ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:43.190022771Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 342.276322ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:43.330542353Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 564.13143ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:43.6830115Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 903.443871ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:44.256930705Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:44.257569702Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"b560bb82-c230-4138-8c6c-75918fade9dc","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":543,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 1.225049633s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:45.170918688Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 798.452048ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:46.406030199Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 1.346076341s","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:47.214986924Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:48.128470754Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Started propolis in zone: oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:48.590566726Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:1416"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 53.088114ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:48.599827135Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 133.166628ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:48.664244926Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 191.104994ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:48.80639731Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\") failed: Property not found. retry in 245.192322ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:27:49.007803783Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.263905585Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:1425"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.30288839Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:1439"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABGdGtcWFxYAAB0a1xYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAARnRrXFhcWAAAdGtcWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiMTMxNzQ3Zi0zYTlkLTQ4ZDgtOWM5Mi04Zjc2ODEyOGFkY2MiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSnI4ZHBKaFVHekxpTEo1ZTBXd1M5SjExbzZSRzlRN1ZrTnNLc0EzMVRGbSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-14aa8e10166\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 230cb32f-7c99-48aa-bfc0-b23e19f98184, read_only_parent: Some(Volume { block_size: 512, id: 0c3be992-a0a3-45c4-b106-087fb22ccd4c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ed895732-7459-4e8d-baac-5393d9418000, key: Some(\"14qtOIxuHCKGHRjgQ2G4QxGZjDCCs5BPzi0wqTKoCtQ=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 230cb32f-7c99-48aa-bfc0-b23e19f98184, key: Some(\"dchOU53ju6AuCY+zZ6G8pD2HufppaHs+QlUJ6aIyAVM=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: b131747f-3a9d-48d8-9c92-8f768128adcc, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.303485745Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:681"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.460596906Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.461083158Z","hostname":"buskin","pid":1671,"uri":"/instances/b131747f-3a9d-48d8-9c92-8f768128adcc/state","method":"PUT","req_id":"bfdc6998-548a-464c-a8a8-d0c2a128fa34","remote_addr":"[fd00:1122:3344:101::a]:53249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8626250,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-02-28T13:27:49.461543560Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.470329026Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.470347847Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-28T13:27:49.461543560Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }","propolis_id":"30be6ba5-dde8-4022-8eb5-6855a07f4c69"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.470955533Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }, propolis_id: 30be6ba5-dde8-4022-8eb5-6855a07f4c69, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-28T13:27:49.461543560Z } }","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-02-28T13:27:49.537025276Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.537045696Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.537066277Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-28T13:27:49.537025276Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }","propolis_id":"30be6ba5-dde8-4022-8eb5-6855a07f4c69"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:49.537076847Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }, propolis_id: 30be6ba5-dde8-4022-8eb5-6855a07f4c69, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-28T13:27:49.537025276Z } }","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.207707626Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33440"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.208586569Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0c23f691-993b-4356-9ce2-dd0473784193, kind: Service { id: 0ed061a1-d644-4f36-95d9-99c559521914 }, name: Name(\"external-dns-0ed061a1-d644-4f36-95d9-99c559521914\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 32])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 546fa748-2a0b-4e62-98d9-13dc37cb9555, kind: Service { id: de116760-7ce8-4051-89b5-dbe725a06843 }, name: Name(\"external-dns-de116760-7ce8-4051-89b5-dbe725a06843\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 188, 245])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 07375275-2e40-4b98-95ca-a28595593c4b, kind: Service { id: b05778b9-9d7a-46ab-9333-eb3104c321fe }, name: Name(\"nexus-b05778b9-9d7a-46ab-9333-eb3104c321fe\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 44])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: dbb6ff29-3ecd-44b2-9a15-f59c94d4a2ff, kind: Service { id: 2063053f-6bdd-4615-8127-179923ae9a65 }, name: Name(\"nexus-2063053f-6bdd-4615-8127-179923ae9a65\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 2])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ee43af6d-e98e-4ac6-b74c-07207b78abdb, kind: Service { id: de28215d-5e75-4ba0-aaaf-7fc4933ba4c2 }, name: Name(\"nexus-de28215d-5e75-4ba0-aaaf-7fc4933ba4c2\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 191])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.20864617Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.208684991Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.208694562Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.208713462Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.208730522Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.208750183Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.208770623Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3fad2251-3893-4ac8-a942-da64402ec764","remote_addr":"[fd00:1122:3344:101::c]:33440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1022,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25876c8c-3919-4b7d-a2bd-afdaf82ce9c3\", \"content-length\": \"127\", \"date\": \"Wed, 28 Feb 2024 13:27:53 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"25876c8c-3919-4b7d-a2bd-afdaf82ce9c3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.277130205Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.444020338Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52866"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.444246464Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"73bb4bba-0c0b-4d55-a419-b0765b1efb8b","remote_addr":"[fd00:1122:3344:101::c]:52866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":204,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.445146847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.445186338Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.451748917Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:27:53.451991984Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"e7cda21d-f3f0-4e51-9ff5-0886690fe7a0","remote_addr":"[fd00:1122:3344:101::c]:52866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7584,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:02.225617257Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:02.225684599Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"33fba07f-1878-43fd-adc5-e7443bd1ecf0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54227","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.252476459Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/2c331470-b440-4c33-9aa1-35acb298a49d","method":"GET","req_id":"aafc8a1a-2e11-4d53-ac2b-c0ae9953144e","remote_addr":"[fd00:1122:3344:101::d]:38246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":344,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.727823854Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.728983613Z","hostname":"buskin","pid":1671,"uri":"/instances/b131747f-3a9d-48d8-9c92-8f768128adcc/state","method":"PUT","req_id":"8193e926-17cf-40c0-a358-2fdee1dc429f","remote_addr":"[fd00:1122:3344:101::a]:65217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1121,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-02-28T13:28:14.729046325Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.729069346Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.729077326Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-28T13:28:14.729046325Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }","propolis_id":"30be6ba5-dde8-4022-8eb5-6855a07f4c69"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.729089626Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }, propolis_id: 30be6ba5-dde8-4022-8eb5-6855a07f4c69, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-28T13:28:14.729046325Z } }","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopped), migration_status: NoMigration, time: 2024-02-28T13:28:14.753526246Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.753544707Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.753563077Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-28T13:28:14.753526246Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }","propolis_id":"30be6ba5-dde8-4022-8eb5-6855a07f4c69"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.753586878Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(30be6ba5-dde8-4022-8eb5-6855a07f4c69), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-28T13:27:40.459888828Z }, propolis_id: 30be6ba5-dde8-4022-8eb5-6855a07f4c69, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-28T13:28:14.753526246Z } }","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-02-28T13:28:14.911675163Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.911711744Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.911742125Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-28T13:28:14.911675163Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-28T13:28:14.911675163Z }","propolis_id":"30be6ba5-dde8-4022-8eb5-6855a07f4c69"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.911760105Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:602","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.911912609Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.912272969Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:14.91231086Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:15.495786752Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1235","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69\", bundle_id: a9ac457d-eca4-4c52-a0b9-721bc78457f4 }, time_created: 2024-02-28T13:28:14.912524285Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_30be6ba5-dde8-4022-8eb5-6855a07f4c69","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:28:15.495878794Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:776"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:17.069915173Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:17.095564374Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-28T13:28:14.911675163Z }, propolis_id: 30be6ba5-dde8-4022-8eb5-6855a07f4c69, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-02-28T13:28:17.095504112Z } }","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:17.1714643Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:38591"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-02-28T13:28:17.173285527Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:17.173296198Z","hostname":"buskin","pid":1671,"uri":"/v2p/cb015a45-bd55-49a7-878e-451e1a99bc53","method":"DELETE","req_id":"2b1bd5ce-d342-4309-8a81-e5704b19fd08","remote_addr":"[fd00:1122:3344:101::c]:38591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1670,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:17.32585513Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-28T13:28:14.911675163Z }, propolis_id: 30be6ba5-dde8-4022-8eb5-6855a07f4c69, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-02-28T13:28:17.095504112Z } }","instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-02-28T13:28:17.328587891Z","hostname":"buskin","pid":1671,"instance_id":"b131747f-3a9d-48d8-9c92-8f768128adcc","component":"InstanceManager","file":"sled-agent/src/instance.rs:719"}