[ Mar 13 04:48:30 Enabled. ] [ Mar 13 04:48:30 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Mar 13 04:48:30 Method "start" exited with status 0. ] [ Mar 13 04:48:30 Rereading configuration. ] [ Mar 13 04:48:30 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:30.617831211Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:248"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:30.671474245Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:30.671499995Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:30.785101302Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:30.79396194Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:30.802051036Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:265","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.407114037Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.553054686Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.553076617Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.553161208Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55326797Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55329112Z","hostname":"sock","pid":1671,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55329707Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.553347251Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.556380528Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:31.556387968Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.556396789Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.556401369Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.556441859Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.556453959Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55646399Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55646967Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55647491Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55648256Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55649023Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55649548Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55650126Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:244","underlay_network_info":"None"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55651244Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.5565203Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.556525771Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.55653106Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.556538901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.747380139Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.74742793Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.74743778Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919609208Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919654128Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919666328Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919674079Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919680729Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919686069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919691479Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919705609Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919711629Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919720919Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919725629Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.91973994Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919798581Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 494592, free: 20937470976, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919827041Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919918392Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919949423Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919955203Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919967873Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919989384Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.919994384Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.920035274Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.920048914Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.934490599Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:700"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.93451395Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999931509s","next_cleanup":"Instant { tv_sec: 1965, tv_nsec: 343335796 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.93455728Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3531"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.934625101Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.936733124Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.936813195Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.937711669Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.93779143Z","hostname":"sock","pid":1671,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.937986344Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.937995344Z","hostname":"sock","pid":1671,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938070275Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938147066Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938178196Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938182787Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938214187Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938241237Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938278948Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938295698Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938299398Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938302838Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938323679Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938353569Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938622463Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938761625Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938783986Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938835237Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.938846017Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.93903603Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939160362Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939206763Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, rack_id: db9a57fa-f70a-4fe0-9bbc-f139e390d48e, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939325594Z","hostname":"sock","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:54482","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, rack_id: db9a57fa-f70a-4fe0-9bbc-f139e390d48e, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939432776Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939442196Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939450136Z","hostname":"sock","pid":1671,"file":"sled-agent/src/server.rs:47"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939473196Z","hostname":"sock","pid":1671,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939480107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.939483277Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.94229678Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3762"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.948181502Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.948220353Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.948224753Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.948252253Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.948339125Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.948343614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.978662286Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:31.987106937Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.021032965Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:356"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.021066375Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.093146356Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.115558095Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.115609926Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.115623746Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.115673417Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.115791578Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.115806279Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.142424193Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.149670615Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.254615648Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.277175108Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.277211949Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.277218139Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.2772443Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.277376742Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.277383012Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.305362687Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.312856914Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.417269498Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.440193514Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.440235675Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.440245325Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.440282145Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.440391457Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.440399967Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.467880905Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.475587614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.581632424Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.604317167Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.604361847Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.604370768Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.604387188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.604508659Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.60451588Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.63220438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.640048832Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.740561786Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.762427836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.762463466Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.762471147Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.762500087Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.762595748Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.762600949Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.790265979Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.797946328Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.907504502Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.93048526Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.930561341Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.930573461Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.930586971Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.930699633Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.930706803Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.959174836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:32.966017912Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.073408423Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.096142936Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.096179887Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.096210277Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.096221127Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.09636176Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.096368349Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.125136077Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.132828447Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.238595862Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.261862843Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.261918154Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.261928184Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.261972075Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.262109637Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.262125478Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.289824649Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.297260304Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.400989467Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.416520539Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.416550529Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.41655796Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:368"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.421770441Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 1721856, free: 20936243712, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 775680, free: 20937189888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 771072, free: 20937194496, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 775680, free: 20937189888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 494592, free: 20937470976, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 775680, free: 20937189888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 1728000, free: 20936237568, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 780288, free: 20937185280, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 771072, free: 20937194496, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.43970502Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.441950644Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.457377844Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.46417972Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.464213971Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.469197118Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:383"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.469212708Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.469219779Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.532417771Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.751136863Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.764118405Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.764248777Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.764303218Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.764342819Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:33.937955129Z","hostname":"sock","pid":1671,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.480178912Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.533204907Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.533254137Z","hostname":"sock","pid":1671,"component":"VmmReservoirManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:252"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.533263788Z","hostname":"sock","pid":1671,"component":"VmmReservoirManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:260"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.533355079Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:982","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.533361909Z","hostname":"sock","pid":1671,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534334514Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534359185Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534401355Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534413235Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534648759Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534850422Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534870043Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:244","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534879143Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3531"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534884413Z","hostname":"sock","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:54482","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534950764Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.534956454Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.733434411Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.733511262Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.733545783Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.733593364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.73404908Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1138","retry_after":"331.534716ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.738931867Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.740010723Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.740902917Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.741995874Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.742973249Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.74433608Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.744454942Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.744768627Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.745041751Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.745352266Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.745549859Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.745683781Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.745816074Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.745931595Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.746009086Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746043357Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746152909Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746259941Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746362502Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746469704Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746589355Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746704687Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.74684139Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.746967341Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.747069083Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.747082713Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.747191945Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.747316757Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.747429499Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.74753843Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.74879199Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.750218912Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.751586853Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.753017646Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.754370117Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.756790424Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.758233847Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.759638558Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.76102737Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.762533234Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.763992456Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.765413738Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.767267757Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63592"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.76741231Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.76744813Z","hostname":"sock","pid":1671,"uri":"/zpools","method":"GET","req_id":"9887e53c-e6cf-4fa6-b7e5-2d111cea9edd","remote_addr":"[fd00:1122:3344:101::1]:63592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":82,"response_code":"200"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.767668223Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.769573353Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.770956954Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.772414167Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.774866295Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.776322638Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.778754646Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.780213529Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.781697412Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.783122274Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.784556456Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.786039469Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.787459581Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.788846283Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.790311316Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.791717997Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.793098529Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.79448592Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.795935413Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.796662384Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39047"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.796762696Z","hostname":"sock","pid":1671,"uri":"/sled-role","method":"GET","req_id":"f4b689d0-af9b-4033-bba1-8c5aab5221f1","remote_addr":"[fd00:1122:3344:101::1]:39047","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.797091491Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.798639375Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.79959604Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.799640281Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.799954525Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.800021707Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.80024268Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:741"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.80150062Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.802932012Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.804388085Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.805930898Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.80795795Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.809488634Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.811483735Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.812873706Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.814308209Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.815785172Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.817639161Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.819001872Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.820439514Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.821849446Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.823318509Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.824728591Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.826158513Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.827663847Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.829628287Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.831255352Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.832123826Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.832299509Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58784"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.832581063Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:34.832586453Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.927896715Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:34.933199708Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.120325298Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1138","retry_after":"524.16455ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.1204687Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.121962484Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.123422276Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.124813538Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.125943695Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.126946841Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.127998478Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.128989203Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.129831996Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.131295409Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.131873638Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.133163668Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.13460232Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.136021722Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.137387944Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.323237874Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.324782158Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.326007417Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.326899011Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.329294908Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.329963789Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:48:35.560558955Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.564229462Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.565405821Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.567263879Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.568311766Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.569458394Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.571402404Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.573477956Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:35.5743877Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:35.574399561Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.788780755Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1138","retry_after":"943.968565ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.788917607Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.790588073Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.792409491Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.989248133Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.990558853Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.992146978Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.993285615Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.994049917Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.995477189Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.996962742Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.997902167Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:35.998990064Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.00000608Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.001148167Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.00262511Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.004118554Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.005595067Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.007028219Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.183685507Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.185092479Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.186017303Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.187309923Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.188011564Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.189229723Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.189961704Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.395324598Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.397502332Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.398921984Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.39995726Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.401004817Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.401879Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.402935907Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.404578322Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.406028875Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:36.406289709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:36.406297679Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.608071857Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:48:36.608696937Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.609270846Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.609956256Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.611163135Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.840063275Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.842016116Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1138","retry_after":"2.776352466s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.84229157Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.843265985Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.844790799Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.846049548Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.846942922Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.847972778Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.848939593Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.849950249Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.85131225Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.85189427Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.852863894Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.854229686Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.855659818Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.857010539Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.858478222Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:36.859898264Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:37.062958752Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:37.064023809Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:37.064944793Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:37.066417066Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:37.06795543Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:37.069482834Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:37.267867249Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:37.456698216Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:37.456763567Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:37.456805578Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:37.456890759Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:37.45698239Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 42.633606ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.103864591Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:48:38.513625934Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 90.507878ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.513705475Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.513714345Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.515068257Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.516045282Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.516878355Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.518000802Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.518944007Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.519892912Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.520797456Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:38.521815931Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:38.522765966Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.257634986Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 195.405541ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.258905745Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.259708928Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.26113051Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.261854461Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.26304312Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.264112466Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.265225644Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.266636795Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:39.2669528Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.447259765Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.44825745Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.449794284Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.451778965Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.453293599Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.454060601Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.856400588Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:48:39.856438259Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 434.366032ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.856598001Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.85712015Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1138","retry_after":"3.001271787s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.858133225Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:39.859665149Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.063475489Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.065436779Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:40.506380927Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.696821569Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 821.981815ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.697033693Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.921044737Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:48:40.921088067Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.92318Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.924160295Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:40.924984518Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:41.127993005Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:41.130261931Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:41.131218346Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:41.320996117Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:41.321923662Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Installing Omicron zone: oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:41.322072114Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:41.775311763Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 766.014486ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:41.99695148Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:41.999156374Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:48:42.000711139Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:42.256131621Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:42.257827868Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:42.259029486Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:42.260049652Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:42.519528088Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:42.521383677Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:42.523363978Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:48:43.459134772Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:43.459283974Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1138","retry_after":"5.524598551s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:43.460001885Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:43.90781442Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:44.837724153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2284"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:45.049995674Z","hostname":"sock","pid":1671,"component":"VmmReservoirManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:264"} {"msg":"Profile for oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:45.051824703Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Installing Omicron zone: oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:45.052066706Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:45.057769865Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:45.057933618Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:45.063056377Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Installing Omicron zone: oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:45.079001905Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 44.376329ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:45.196749227Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 96.775755ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:45.249801542Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 144.37818ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:45.446036944Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 338.64005ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:45.599676793Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 1.179969093s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:45.947442722Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:47.154858141Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:47.633503516Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:47.633557637Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:47.633582717Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:47.633933663Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 1.177862491s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:47.650394459Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:47.650448829Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 54.899416ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:47.759676328Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 72.426172ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:47.768972023Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 133.16938ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:47.828544769Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 105.740755ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:47.963071822Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 245.880001ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:47.978884427Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 278.702424ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:48.090480823Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 370.007602ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:48.249701509Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 587.075031ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:48.382107719Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 738.876383ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:48.628678214Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 886.921191ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:48.837565143Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 1.17869841s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:48.979130024Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 1.127313236s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:49.378107109Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230\") failed: Property not found. retry in 659.198642ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:49.735841353Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 645.957786ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:50.167937764Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 692.746274ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:50.515056093Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 675.114644ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:50.822392813Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 1.262771008s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:51.216513492Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b\") failed: Property not found. retry in 1.34219115s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:51.505406645Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620\") failed: Property not found. retry in 512.848896ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:52.488591237Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:52.964080192Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2301"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:52.964116013Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.024568123Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_internal_dns_615a9669-97b2-4c06-9ca1-f5cee9e02620"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.024612584Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_internal_dns_008b60d6-d8b4-4d18-b06a-f3d5490b3230"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.024620614Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_internal_dns_69e66ea6-9cdf-49f5-aa83-36e00f886d4b"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.024662954Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.024894378Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.025150422Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d5245e87-93b5-415a-b84c-d4f5561ff474","remote_addr":"[fd00:1122:3344:101::1]:58784","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":18192732,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.026897539Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"failed to write DNS configuration (will retry in 122.483315ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:53.427192825Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:437","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe2d:d20d","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.519358348Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2308"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.525040487Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2369"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.568603154Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.614413047Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.6493589Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2554"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.660026466Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.660048157Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.704922854Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:53.900821151Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2718"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:54.088977638Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2433"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:54.384811728Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2468"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:54.581520478Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2756"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:54.729637782Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2761"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:54.907840003Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2662"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.07931828Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2462"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.089948635Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3786","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.089975016Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3802"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.089979416Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.636270573Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3810"} {"msg":"Failed to query MGS at fd00:1122:3344:101::2: Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146): error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146): error trying to connect: tcp connect error: Connection refused (os error 146): tcp connect error: Connection refused (os error 146): Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:55.640479788Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Failed to look up switch zone locations","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:55.640532449Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:223","retry_after":"84.503689ms","error":"No switch locations found"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.642637042Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.726171921Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.767305511Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Failed to query MGS at fd00:1122:3344:101::2: Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146): error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146): error trying to connect: tcp connect error: Connection refused (os error 146): tcp connect error: Connection refused (os error 146): Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:55.772114305Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Failed to look up switch zone locations","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:55.772147736Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:223","retry_after":"78.203697ms","error":"No switch locations found"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.77816923Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3928"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.852278112Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.874177003Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3909","rack_subnet":"fd00:1122:3344::"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.90164792Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Failed to query MGS at fd00:1122:3344:101::2: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2f8caa6-ca38-425c-95d5-115448b862fe\", \"content-length\": \"152\", \"date\": \"Wed, 13 Mar 2024 04:48:55 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"f2f8caa6-ca38-425c-95d5-115448b862fe\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:55.903348397Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Failed to look up switch zone locations","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:55.903369077Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:223","retry_after":"187.823193ms","error":"No switch locations found"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:55.928126792Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 58.958411ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:55.960203251Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1fafb67c-7fe5-44df-9bdd-a3143892bcc6\", \"content-length\": \"152\", \"date\": \"Wed, 13 Mar 2024 04:48:55 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"1fafb67c-7fe5-44df-9bdd-a3143892bcc6\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 138.462847ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:56.020846134Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"941adc27-5eeb-493a-8655-684bd6da2caa\", \"content-length\": \"152\", \"date\": \"Wed, 13 Mar 2024 04:48:55 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"941adc27-5eeb-493a-8655-684bd6da2caa\" }"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.091540473Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.123642843Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Failed to query MGS at fd00:1122:3344:101::2: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b3c36159-f9d7-486a-944e-4eb57a04bc2e\", \"content-length\": \"152\", \"date\": \"Wed, 13 Mar 2024 04:48:55 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"b3c36159-f9d7-486a-944e-4eb57a04bc2e\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:56.124433595Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Failed to look up switch zone locations","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:56.124440335Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:223","retry_after":"198.392636ms","error":"No switch locations found"} {"msg":"Failed to get switch ID from MGS (retrying in 248.487877ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:56.160520836Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"89a4c4e6-e4f5-44d6-8054-dd1ac20abb3d\", \"content-length\": \"152\", \"date\": \"Wed, 13 Mar 2024 04:48:55 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"89a4c4e6-e4f5-44d6-8054-dd1ac20abb3d\" }"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.323925368Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.355848894Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.357643952Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.357649792Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.3871113Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.387279653Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45993"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.387468706Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.387476966Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.408036776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.408086517Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.411122734Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.428390542Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.428423663Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.447655462Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.447711493Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.447906066Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.447995417Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.455279781Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.455357402Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.455699417Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.455712447Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.488758341Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.534688736Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.536068827Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.538546446Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.538554716Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.585513216Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 225, 173]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.585580747Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1276","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.586211827Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.586221117Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.586455641Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.588134217Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.612183071Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:56.630310833Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:57.761036419Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:57.761184872Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 70.733015ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:57.896837291Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 89.974402ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:57.986334623Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 212.886389ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:58.106269138Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 246.837708ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:58.331886377Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 1.126709184s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:58.589496354Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:59.670133431Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62637"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:48:59.670279724Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5c1d11d8-d962-429c-b16d-286593e621c2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62637","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":63,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 1.107577341s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:48:59.726406697Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 1.292100812s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:00.843380569Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5\") failed: Property not found. retry in 534.50338ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:02.145570272Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.702812489Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.702859499Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.703115223Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.703333767Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7e829b3e-8c0d-4a03-adc7-d5cb8d381546","remote_addr":"[fd00:1122:3344:101::1]:45993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6315981,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.705363188Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.734642473Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.734774156Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45007"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.74405576Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3517"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.744157301Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"4782a9f9-aa07-4837-aae3-b51705647ba0","remote_addr":"[fd00:1122:3344:101::1]:45007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9259,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:02.746193383Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.832622448Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:02.832719749Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60738"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:04.841266458Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3517"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:04.841496972Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"693737f8-9f8c-460c-b2de-c8cf6e8ba14b","remote_addr":"[fd00:1122:3344:101::1]:60738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":2008690,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:04.844393697Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:04.985837096Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:04.985987409Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:62455"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:04.996027145Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3517"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:04.996152527Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"75b59d17-72a8-40a5-aa5c-ce6d90d0c620","remote_addr":"[fd00:1122:3344:101::1]:62455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10116,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:04.998417082Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:05.128136499Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:05.128246251Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33962"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_483f2279-0a3f-444e-ac3d-24db30026ca5': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.137418506Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3517"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.137539748Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"0c2055b6-0909-44b0-bb5a-3ac9ceadcdf5","remote_addr":"[fd00:1122:3344:101::1]:33962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009242,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:09.141480749Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.288676968Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.28880044Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:56664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.299534767Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"723ee5ee-ba69-4b14-83e1-073c9e040f42","remote_addr":"[fd00:1122:3344:101::1]:56664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":10655,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.302099867Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:09.302104937Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.448740228Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.44889284Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51572"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.457278001Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"de10b1ae-0dde-4fe8-bd1a-b538935c478e","remote_addr":"[fd00:1122:3344:101::1]:51572","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8338,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.460678194Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:09.460683034Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.780304385Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.780443917Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.788165667Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"4cf0d8fc-771c-437e-a766-295692901bb7","remote_addr":"[fd00:1122:3344:101::1]:51603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7664,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:09.790181459Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:09.790198719Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.040157636Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.040243148Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48031"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.048052079Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"9483a2f8-adc9-433a-a9a5-6ece9ee747a3","remote_addr":"[fd00:1122:3344:101::1]:48031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7758,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.050084341Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:10.05008891Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.541853549Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.541993131Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.55088123Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"efc2aa13-ce4a-47e0-bbaf-590f381202a3","remote_addr":"[fd00:1122:3344:101::1]:33977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8824,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:10.553210806Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:10.553221186Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.077373298Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.07748534Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58113"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.086190725Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"c298758f-8222-47a6-b101-d2d2cf8a5f82","remote_addr":"[fd00:1122:3344:101::1]:58113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8617,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.089878082Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:11.089884533Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.686588603Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.686712025Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42649"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.694566917Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"daf4e3a1-66c3-4e8a-99eb-3f4943dab390","remote_addr":"[fd00:1122:3344:101::1]:42649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7699,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:11.697510103Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:11.697524953Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.099096379Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.099206651Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.107107323Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"eea86e9f-a90d-4f5a-8833-2f82673fe727","remote_addr":"[fd00:1122:3344:101::1]:35676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7835,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.109792325Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:12.109800275Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.636423456Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.636527108Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.644195237Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"83b537f2-d174-4a3c-8536-d5c0347198d5","remote_addr":"[fd00:1122:3344:101::1]:54557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7596,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1710305347.3875587, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:12.646621434Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:12.646627925Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.50535292Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.505500002Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64634"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.50924148Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.527122949Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"2213bf82-588d-4cb8-bfed-a02bee28655e","remote_addr":"[fd00:1122:3344:101::1]:64634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":21481,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1710305353.2648702, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.529468645Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.529474425Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.558253183Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.558416345Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61479"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.558654669Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.558664199Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.579193928Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.579230689Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.598350746Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.598387817Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.616379927Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.616414217Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.670967896Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.671004546Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.72396258Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.72399728Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.773320658Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.773360538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.825215315Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.825253995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.875788791Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.875837952Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.876065276Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.876182907Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.961377703Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.961420333Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.961424713Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.961428073Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:13.961440264Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.054696724Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.058568394Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.062670628Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.067206858Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.071998483Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.076757937Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.08144114Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.085792698Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.089555436Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:14.093872953Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.958878217Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.959038159Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.959158201Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.963064032Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.965863976Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.987270928Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.987487132Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.99509133Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:17.995248393Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:18.053639791Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:18.053773403Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 35.17204ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.112965423Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 40.443329ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.123149281Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 62.99813ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.133228948Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 33.487203ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.141360505Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 79.945158ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.156160575Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 93.358236ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.174192245Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 101.211333ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.271811863Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 50.414293ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.340550503Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 210.386279ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.407002586Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 182.489144ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.431297424Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 261.678822ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.487868664Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 259.922521ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.543601941Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 35.246895ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.591281042Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 445.148901ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.659628505Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 134.149743ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.70425786Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 305.783042ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.749218709Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 346.925912ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.793207733Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 530.789599ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.83675698Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 180.825303ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:18.872585527Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 378.135898ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:19.425755031Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 1.045878735s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:19.460776635Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 644.218702ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:19.576527076Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 410.800043ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:19.626816838Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 572.423053ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:19.697628639Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 407.826735ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:19.763341231Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 1.137640827s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:20.26023314Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 889.973214ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:20.387334746Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 592.749899ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:20.519292229Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 1.171562903s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:20.534965382Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 1.329476267s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:20.552471395Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 1.291422679s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:21.12507442Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 1.151639457s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:21.288926199Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 993.194225ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:21.407531883Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 1.281045422s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:21.716947656Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 513.382783ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:21.892457375Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 606.363469ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:22.411395186Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 1.292288726s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:22.418645149Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 719.557604ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:22.426739085Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 817.013087ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:22.460779894Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 1.474674425s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:23.009070302Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 1.497227928s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:23.029733854Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 1.243557062s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:23.15746578Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 1.394429403s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:23.288658841Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 622.221414ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:23.721822848Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 761.799246ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:24.353580084Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 1.126843247s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:24.411426723Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 581.214901ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:24.492626536Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 791.010382ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:24.537013486Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 1.333959097s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:24.691608791Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 944.969722ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:25.082794835Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 1.479580726s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:25.127269486Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 1.177879264s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:25.337651978Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 506.51058ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:25.547722106Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 685.388651ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.036036201Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 620.298868ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.045392536Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 904.393999ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.06298191Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 779.260599ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.525671036Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c\") failed: Property not found. retry in 1.147964938s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.616816033Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728\") failed: Property not found. retry in 1.289295742s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.675350904Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 1.015379747s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.731289654Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52\") failed: Property not found. retry in 1.439704689s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:26.97661274Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f\") failed: Property not found. retry in 847.097992ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:27.314303951Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a\") failed: Property not found. retry in 631.690451ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:27.758016433Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.436634467Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_cockroachdb_24d8c7e8-c65a-4393-9f69-957caf4c073f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.436683988Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_cockroachdb_e35710c2-0ad4-4310-b535-49fbc3b2e728"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.436690238Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_cockroachdb_f0953b77-046f-4954-9588-57eed328d77a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.436695898Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.436700398Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_cockroachdb_891ec291-a154-4e9a-b484-c05219c7ab52"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.436744839Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.437083004Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.437317268Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d3ba89f6-9cb1-4db4-af77-abf1b8ca8828","remote_addr":"[fd00:1122:3344:101::1]:61479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":14444818,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.470295471Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45399"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.503361085Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"56e6053a-8646-4776-883c-a4ffd390d055","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":32851,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:28.503632059Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"124.025137ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"56e6053a-8646-4776-883c-a4ffd390d055\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"56e6053a-8646-4776-883c-a4ffd390d055\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.656809032Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"87995d89-e1e8-49f9-ae1a-6e310928afa3","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27758,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:28.657056655Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"157.9524ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"87995d89-e1e8-49f9-ae1a-6e310928afa3\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"87995d89-e1e8-49f9-ae1a-6e310928afa3\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:28.8431912Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"83b7d84d-de52-4961-ae53-f7e9c3bfb6f5","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27360,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:28.843437254Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"147.723163ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"83b7d84d-de52-4961-ae53-f7e9c3bfb6f5\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"83b7d84d-de52-4961-ae53-f7e9c3bfb6f5\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:29.022003432Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"35364416-cd3a-4eff-a775-383834ece8b4","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":29917,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:29.022250065Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"128.01933ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35364416-cd3a-4eff-a775-383834ece8b4\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35364416-cd3a-4eff-a775-383834ece8b4\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:29.177589151Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"ee0f5be0-0ad9-4aed-bdd7-5b96cb676b1c","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26817,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:29.177841345Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"198.865184ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ee0f5be0-0ad9-4aed-bdd7-5b96cb676b1c\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ee0f5be0-0ad9-4aed-bdd7-5b96cb676b1c\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:29.40511022Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"9e2aa81b-0dcc-43d1-9f39-acd959122e7d","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27068,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:29.405315813Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"259.676529ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e2aa81b-0dcc-43d1-9f39-acd959122e7d\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9e2aa81b-0dcc-43d1-9f39-acd959122e7d\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:29.693399464Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"f96d2051-9e57-41be-acfb-aa8ccafade7b","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26940,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:29.693653598Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"169.64849ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f96d2051-9e57-41be-acfb-aa8ccafade7b\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f96d2051-9e57-41be-acfb-aa8ccafade7b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:29.790609406Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52947"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:29.790687767Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"0206e528-4831-45ee-a2bf-2ebedd356e52","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52947","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:29.892290157Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"7888e1d4-2781-4f15-9141-ed0a1a53dd98","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28551,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:29.89249522Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"236.69768ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7888e1d4-2781-4f15-9141-ed0a1a53dd98\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7888e1d4-2781-4f15-9141-ed0a1a53dd98\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:30.158789932Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"0a6b7f4b-e5f8-425c-8907-3dba50fc0d1a","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28832,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:30.159116097Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"271.767542ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0a6b7f4b-e5f8-425c-8907-3dba50fc0d1a\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0a6b7f4b-e5f8-425c-8907-3dba50fc0d1a\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:30.458356241Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"a522a720-fa62-4227-9637-d32ac9d83067","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26363,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:30.458640145Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"522.181909ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a522a720-fa62-4227-9637-d32ac9d83067\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a522a720-fa62-4227-9637-d32ac9d83067\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:31.010510799Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"48b094c1-e36e-445b-a07f-aeeaa7430140","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_16bef80b-4497-4091-b795-b5147cfeb79c /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28419,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:49:31.010818054Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"849.257975ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"48b094c1-e36e-445b-a07f-aeeaa7430140\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:49:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"48b094c1-e36e-445b-a07f-aeeaa7430140\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:31.8914237Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3351"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:50.346606113Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3369"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.300660525Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3392"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.303078613Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"7f9a0ac1-b22e-4cc9-87fc-19c29f8d9269","remote_addr":"[fd00:1122:3344:101::1]:45399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":27442383,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.377309637Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.378920332Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58772"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.380492777Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.380502007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.406314268Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.40637752Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.426690155Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.426754956Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.444700365Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.444752496Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.464759048Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.467145645Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.483483069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.483540189Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.502855Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.502909941Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.521952747Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.522004948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.539956647Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.541935588Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.631671293Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.631736814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.722845682Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.722899432Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.803953243Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.804018344Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.832185472Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60315"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.832765451Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"6030cb34-0340-4ae3-8e2c-e32386f93c33","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60315","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":457,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.913182662Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:49:59.913236753Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.016988016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.019218611Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.119800685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.119858036Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.23322162Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.23328223Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.331581799Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.33163808Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.439580699Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.441997107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.530135327Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.530194478Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.62930932Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.629376131Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.736332314Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.736395846Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.736825122Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.737211228Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860198621Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860245842Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860251272Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860255802Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860260192Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860265372Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860269502Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860281562Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860286172Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860290422Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860294662Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860299042Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860303123Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860307683Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860312793Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860317393Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860322573Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860466395Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.860472195Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.860621107Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"38ef0b4c-84bd-4adb-b0e3-23fc747530a0","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized)]","latency_us":1481370,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:00.86140626Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"119.522779ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38ef0b4c-84bd-4adb-b0e3-23fc747530a0\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"38ef0b4c-84bd-4adb-b0e3-23fc747530a0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.982222879Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.983892485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:00.983905745Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.009002055Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.009068127Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.031971143Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.032030473Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.053357915Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.053411696Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.073304045Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.073353436Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.093344007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.093397798Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.112427454Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.112480355Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.132345584Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.132398674Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.151435981Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.151593673Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.171274279Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.17132616Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.190260414Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.190319515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.20989096Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.209945691Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.229463624Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.229517945Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.248789315Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.248848416Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.26778906Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.267846901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.287482617Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.287538267Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.307195823Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.307249454Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.326725207Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.326774717Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.346525505Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.346597176Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.367148456Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.367200926Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.388451677Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.388522688Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.388732191Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.388899164Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489784783Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489835554Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489843164Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489847104Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489850744Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489860544Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489864254Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489867884Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489871334Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489874954Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489938625Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489949515Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489954246Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489957906Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489961576Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489966116Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489970826Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489975206Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.489979476Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.490145998Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1216b911-ea53-46cb-894c-359deaaa72fc","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized)]","latency_us":507308,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:01.490623276Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"88.669375ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1216b911-ea53-46cb-894c-359deaaa72fc\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1216b911-ea53-46cb-894c-359deaaa72fc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.579769763Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.580775668Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.58089373Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.602823291Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.602882812Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.62460083Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.624660031Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.644081873Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.644149174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.662639931Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.662704062Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.681498465Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.681551135Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.701465995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.701511626Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.720946338Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.720993219Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.740749686Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.740809487Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.76153157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.761600811Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.781023842Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.783575462Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.800351323Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.800416394Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.820560337Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.82069831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.841139928Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.841199169Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.860384827Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.860515859Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.879501884Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.879560875Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.899489235Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.899543166Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.919652519Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.91970946Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.939094781Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.939159912Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.957929024Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.957989075Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.977235244Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.977313346Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.978112728Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:01.978521374Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.080944537Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081017558Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081037279Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081042559Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081048229Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081054689Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081061369Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081068269Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081073739Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081079409Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081086139Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081092869Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.08109928Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081103889Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.08111026Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.08111469Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.08112026Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.0811247Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.08114127Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.081204971Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3849b47d-068f-47b4-be5c-c46a0384e8dd","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized)]","latency_us":500971,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.081457365Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"142.230242ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3849b47d-068f-47b4-be5c-c46a0384e8dd\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3849b47d-068f-47b4-be5c-c46a0384e8dd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.224939837Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.22576773Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.22578212Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.248751917Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.248802168Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.269039943Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.269097553Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.291296799Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.291344979Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.312528359Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.31257324Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.333709689Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.333752839Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.355159552Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.355208643Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.375662511Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.375711942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.395029492Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.395077113Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.415010143Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.415053443Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.434619368Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.434666969Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.454100771Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.454147652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.474265564Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.474311945Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.495387223Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.495431814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.514335098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.514375298Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.533591057Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.533633908Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.552331109Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.552378539Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.570744015Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.570808226Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.59035609Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.590398351Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.60896997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.6090101Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.629331946Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.629378887Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.629640691Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.629795003Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.72792516Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.727994681Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728014161Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728020701Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728032801Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728040551Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728046882Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728058732Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728064612Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728071812Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728078092Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728085552Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728097302Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728103452Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728110743Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728133833Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728140233Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728147933Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728155083Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.728399377Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"5a010da5-7114-4782-bb2e-0cc984eccb23","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized)]","latency_us":502792,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:02.728663911Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"203.101897ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a010da5-7114-4782-bb2e-0cc984eccb23\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5a010da5-7114-4782-bb2e-0cc984eccb23\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.933445896Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.934114267Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.934128277Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.955136714Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.955201105Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.975626162Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.975687043Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.994060999Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:02.99411831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.016037121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.016094782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.035481163Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.035626755Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.053484733Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.053542874Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.072906395Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.072973246Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.093750249Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.093898411Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.111301472Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.113370864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.132187107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.134685696Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.149793161Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.149849502Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.168866657Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.168924378Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.189469758Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.189525589Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.210224181Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.210277242Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.229054523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.229108824Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.249235407Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.249290068Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.267452651Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.267507732Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.287759316Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.287814967Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.307058607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.307131688Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.326314056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.326404738Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.326805184Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.327155519Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423764612Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423839913Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423857503Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423864253Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423872304Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423878254Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423885894Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423893504Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423899844Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423910544Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423915464Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423924594Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423935445Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423964925Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423974575Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423980705Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423984925Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423990875Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.423995095Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.424004776Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"dc377a3f-fa85-48a7-9513-d06beda857d6","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized)]","latency_us":490114,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:03.42429898Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"197.047103ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dc377a3f-fa85-48a7-9513-d06beda857d6\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dc377a3f-fa85-48a7-9513-d06beda857d6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.623285705Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.624303691Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.624314121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.643297776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.64542997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.659562329Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.65962957Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.678652726Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.678710777Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.698377073Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.698434554Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.717394539Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.71744901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.735059524Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.735124374Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.755896028Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.755952698Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.77727277Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.777324601Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.796938056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.797005727Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.813789288Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.813844939Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.830820053Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.830874534Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.847192228Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.847259089Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.864870283Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.864926683Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.883922359Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.88397982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.903082427Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.903139238Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.922814864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.925383684Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.941255191Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.941315172Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.961750079Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.96180855Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.981552737Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:03.981609678Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.001436607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.001505848Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.001788382Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.002112097Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098006738Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098060289Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098068229Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098073659Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098078799Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098084139Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.09813715Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.09814864Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098176291Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098185211Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098216092Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098224922Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098233662Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098240562Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098248872Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098258022Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098277053Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098288933Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.098302363Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.098469536Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"4927b5b5-9983-4dfa-bf47-3387db3a81d7","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized)]","latency_us":474380,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.09873186Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"277.453731ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4927b5b5-9983-4dfa-bf47-3387db3a81d7\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4927b5b5-9983-4dfa-bf47-3387db3a81d7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.376686093Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.377341493Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.377356653Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.399765832Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.399819712Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.41958948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.419644321Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.439248356Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.439307837Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.458456284Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.458522896Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.47939458Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.479450801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.498527938Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.498581979Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.517384621Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.517437452Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.537321721Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.537382412Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.557938072Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.558093884Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.577188461Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.577249872Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.596501402Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.596559722Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.614757606Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.614819106Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.633537538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.633592588Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.652761977Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.652818247Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.671699081Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.674308542Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.688929299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.68898503Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.705169742Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.705234453Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.722345479Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.7224022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.742523413Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.742598364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.760763786Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.760845218Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.761104651Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.761272004Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855334757Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855386868Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855394178Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855398398Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855401938Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855409508Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855413338Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855416869Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855435249Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855439049Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855443009Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855446439Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.85553977Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855548471Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855555681Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855563281Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855569091Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855572761Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855578391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:04.855720563Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c3f6af67-8b91-4352-8a51-3c05440cefc8","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized)]","latency_us":478632,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:04.855961707Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"163.668962ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c3f6af67-8b91-4352-8a51-3c05440cefc8\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c3f6af67-8b91-4352-8a51-3c05440cefc8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.020958283Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.021485481Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.021511252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.044310906Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.044354327Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.064062174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.064103304Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.084518532Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.084569602Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.104692746Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.104742586Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.124558264Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.124600445Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.144108299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.144157139Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.162909131Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.162968952Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.181751784Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.181798405Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.201085225Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.201133975Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.222774372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.222821073Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.24195445Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.242002991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.260503229Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.26056028Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.280139514Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.280189945Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.299892662Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.299938782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.321325475Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.321379016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.340609715Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.340654565Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.360069567Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.360108858Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.380595817Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.380649098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.400696049Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.40073874Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.419614153Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.419669174Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.419920588Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.420076141Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518326659Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.51839046Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.51839729Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.51840404Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.5184102Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.51843001Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518436181Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.51844153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518448871Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518465901Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518478551Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518484471Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518490351Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518497342Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518503162Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518510622Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518517732Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518528452Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518535672Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.518638274Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"e1c34517-3d36-424d-949a-2142b2d0fb45","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized)]","latency_us":497356,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:05.518922318Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"235.778521ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1c34517-3d36-424d-949a-2142b2d0fb45\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e1c34517-3d36-424d-949a-2142b2d0fb45\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.755935274Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.756460123Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.756476073Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.779834496Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.779897677Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.801873659Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.80193962Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.822894666Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.822958167Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.842713654Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.842772555Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.862022094Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.862081735Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.882596634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.882648765Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.903448009Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.90349903Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.924058809Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.92411097Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.945027955Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.945088826Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.964739412Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.964795393Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.984895825Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:05.984955676Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.004081254Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.004141015Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.023720709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.02378108Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.045441247Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.045512448Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.066045197Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.066097628Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.086258142Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.086310193Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.105050634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.105119045Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.124630878Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.12468655Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.144712011Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.144849583Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.163623485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.163690156Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.164045422Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.164432098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.260902168Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.260962939Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.260970589Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261002159Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.26101102Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.2610171Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.26102486Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.26103242Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.26103951Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.26104697Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.26105295Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261059031Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261065991Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261073351Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261081001Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261091191Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261096311Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261100461Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261112601Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.261182732Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"adcd2b86-301a-48ec-9d56-60e47722f82e","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized)]","latency_us":504940,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:06.261404236Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"284.02211ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"adcd2b86-301a-48ec-9d56-60e47722f82e\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"adcd2b86-301a-48ec-9d56-60e47722f82e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.547280122Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.548281058Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.548299178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.571477339Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.57155449Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.592223381Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.592288812Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.611194066Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.611253097Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.631430221Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.631488492Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.650423286Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.650478807Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.667917948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.6679865Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.688066772Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.688123793Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.708116034Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.708174174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.72778399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.72783709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.747661479Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.74777471Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.767928804Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.767986855Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.788043187Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.788096728Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.807623111Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.807682612Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.826565026Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.826620357Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.846683139Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.84675607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.865861417Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.865930278Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.885264099Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.88532322Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.90463592Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.904695571Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.923124308Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.923188409Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.942631731Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.942702112Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.942936676Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:06.943368983Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039306735Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039403856Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039417077Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039426397Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039441997Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039450927Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039459737Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039479847Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039488667Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039497768Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039506088Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039513278Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039528698Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039537168Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039552539Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039557949Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039573579Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039578769Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039586989Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.03966676Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"b8b41410-65d3-4ef8-a73f-9ffd73193e46","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized)]","latency_us":491584,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:07.039937034Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"750.618759ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b8b41410-65d3-4ef8-a73f-9ffd73193e46\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b8b41410-65d3-4ef8-a73f-9ffd73193e46\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.791707806Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.792790054Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.792806864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.816178007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.816243868Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.838417243Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.838478954Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.859746155Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.859870087Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.881056756Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.881120027Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.90317116Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.903247641Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.924525482Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.924592363Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.945485068Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.94563124Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.966124279Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.96618239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.988186782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:07.988251523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.009338222Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.009405503Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.031861652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.031918983Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.052529833Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.052602584Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.073306116Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.075869416Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.093840676Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.093900997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.114213832Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.114281994Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.134594309Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.13466287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.154815044Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.154880715Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.175514616Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.175570167Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.195625288Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.195691069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.214239138Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.214333019Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.214725646Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.214920549Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316310926Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316366337Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316378347Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316382237Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316385907Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316391607Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316399597Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316403537Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316464118Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316472628Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316487178Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316496628Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316501748Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316516389Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316524789Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316531259Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316535949Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316540539Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316544839Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.316684731Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"0c690cf0-88d7-465f-9948-5a8f922b69bd","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized)]","latency_us":524269,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:08.316924315Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"592.25972ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c690cf0-88d7-465f-9948-5a8f922b69bd\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0c690cf0-88d7-465f-9948-5a8f922b69bd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.909718075Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.910495007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.910512067Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.933235251Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.933310782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.955325824Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.955393535Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.977271666Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.977332927Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.99686007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:08.996922561Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.018825552Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.021393372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.039108198Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.039166718Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.060311017Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.060368548Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.081042519Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.08109893Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.102659446Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.102722287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.124432774Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.124490715Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.145769906Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.145837847Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.166890335Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.169549806Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.187971433Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.188035563Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.210184228Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.210255239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.232847631Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.232912501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.255659255Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.258349627Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.277363473Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.277425914Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.298618414Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.298676864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.319902225Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.319964026Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.340948232Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.341024913Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.341280247Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.341594262Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448730598Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448782619Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448787879Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448791709Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448796599Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448800139Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448803779Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448807279Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.44881067Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.44881439Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.44882249Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.44883364Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.4488372Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448936061Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448941852Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448946241Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448951642Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448957362Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.448962742Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:09.449215766Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"2e57d622-0667-4873-b81e-28d0defbecb1","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized)]","latency_us":539036,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:09.449446309Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"669.327697ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2e57d622-0667-4873-b81e-28d0defbecb1\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2e57d622-0667-4873-b81e-28d0defbecb1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.120011709Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.120571607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.120589607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.145584776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.145637697Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.166275538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.166322699Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.187190163Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.187234054Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.207107063Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.207146254Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.226354523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.226395283Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.247516762Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.247557872Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.269074877Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.269117438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.289276811Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.289327332Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.310556512Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.310601793Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.332185438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.332234799Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.351804754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.351851284Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.371160845Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.371200005Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.389830075Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.389877016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.410200282Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.410244632Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.428437906Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.428483406Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.447285579Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.447323329Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.465944029Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.46598579Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.485464493Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.485502493Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.504475608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.504530889Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.523720128Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.523770438Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.524039393Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.524318227Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620495833Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620549523Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620562254Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620570564Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620580034Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620587594Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620594104Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620600054Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620614844Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620621105Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620628165Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620634825Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620640525Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620645975Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620651945Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620665165Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620670635Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620677856Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.620683706Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:10.620819158Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"fe5d2be0-3b4e-4b7e-8a87-8427e41caa5d","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized)]","latency_us":500494,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:10.621045161Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"524.568154ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe5d2be0-3b4e-4b7e-8a87-8427e41caa5d\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fe5d2be0-3b4e-4b7e-8a87-8427e41caa5d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.14687862Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.147668012Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.147700662Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.170141691Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.170203792Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.191814968Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.19188658Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.210798254Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.210850804Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.230004773Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.230066043Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.248610452Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.248676383Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.268356149Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.26841061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.287833712Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.287899293Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.306549263Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.306604584Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.327221505Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.329806545Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.3462381Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.346307071Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.365249156Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.365323037Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.384398454Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.384463725Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.403890347Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.403947238Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.42403084Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.424086141Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.445127668Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.445191629Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.465769939Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.46583558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.485656889Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.48571371Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.505315095Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.505531628Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.525944415Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.526008436Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.546997413Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.547119085Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.547332368Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.547499331Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649149752Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649214153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649223383Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649228353Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649236603Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649242283Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649254283Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649260213Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649264573Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649270143Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649274574Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649279003Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649283744Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649292804Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649298314Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649308084Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649312634Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649316184Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649320934Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:11.649404025Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"fba904c7-3ceb-4439-9e22-1db809cbf004","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized)]","latency_us":502194,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:11.649610369Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.557033637s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fba904c7-3ceb-4439-9e22-1db809cbf004\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fba904c7-3ceb-4439-9e22-1db809cbf004\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.20829119Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.209064833Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.209078403Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.231696145Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.231761236Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.251750037Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.251811707Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.271064317Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.273594686Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.291236621Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.291291652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.30917839Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.309234071Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.328327288Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.328380808Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.346046823Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.346120704Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.364703543Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.364783864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.384091795Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.384153276Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.402618813Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.402683264Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.42105765Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.421116571Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.44036909Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.440425701Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.459917934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.459982785Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.4782974Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.480805639Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.498116438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.498174179Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.517719663Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.517782124Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.53743218Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.537499341Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.556319594Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.558901764Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.5760422Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.576103601Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.595069866Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.595144407Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.595576494Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.595873259Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692649354Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692701745Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692708105Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692714395Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692725455Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692731535Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692737015Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692747796Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692754596Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692760716Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692767586Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692774196Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692780446Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692787236Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692793776Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692800246Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692805776Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692812117Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.692816506Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:13.69307355Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8f8004ef-824e-4ccc-ac48-9bff545a74db","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized)]","latency_us":484256,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:13.693280714Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.383828153s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f8004ef-824e-4ccc-ac48-9bff545a74db\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8f8004ef-824e-4ccc-ac48-9bff545a74db\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.078351366Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.079090177Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.079106587Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.101667008Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.101728149Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.121674089Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.124217149Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.142710256Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.142780218Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.161968076Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.162032657Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.181566561Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.181629022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.200441825Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.200516846Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.218623777Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.218676868Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.238415245Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.238473866Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.257604314Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.257667384Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.276585449Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.27666718Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.296303905Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.296362246Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.315215299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.31526698Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.335785419Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.33584246Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.354899467Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.354956098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.373797621Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.373859642Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.393087001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.393146902Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.411506957Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.411571808Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.430529223Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.430595074Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.450046687Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.450107197Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.468752047Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.468814439Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.469458038Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.469627581Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567887989Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.56795257Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.56795791Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567962621Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567968081Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567972051Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567976431Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567980921Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567985871Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567991061Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.567995651Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568001071Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568005821Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568011491Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568016331Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568022551Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568027262Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568043022Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568047722Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:15.568145813Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"eb6fa07f-6a62-4ee3-93e3-fd66972582ae","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized)]","latency_us":489473,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:15.568358017Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.68287326s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb6fa07f-6a62-4ee3-93e3-fd66972582ae\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb6fa07f-6a62-4ee3-93e3-fd66972582ae\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.252021693Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.252778565Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.252795235Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.274762696Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.274832868Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.294395612Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.296816399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.31355904Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.313616231Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.333121544Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.333184235Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.351867946Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.351937507Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.371554892Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.371613483Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.389616833Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.389674204Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.409712265Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.409772526Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.431115518Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.431171139Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.451649307Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.451706868Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.470159075Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.470212016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.488068094Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.488121855Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.507793221Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.51030261Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.525998754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.526060215Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.545630209Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.54568699Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.564129267Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.564183608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.581894183Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.581944854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.600887509Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.6009412Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.619909375Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.619970296Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.638137348Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.638221329Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.638577855Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.638985091Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734303354Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734358535Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734366945Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734372495Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734378445Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734383745Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734399485Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734405345Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734411736Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734425956Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734432386Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734439186Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734445586Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734450976Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734475317Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734482017Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734488697Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734495327Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734501617Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:17.734599838Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"ec4b8faf-bcb7-460a-b9b0-f2a431564e03","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized)]","latency_us":482270,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:17.734824122Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.703498748s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec4b8faf-bcb7-460a-b9b0-f2a431564e03\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ec4b8faf-bcb7-460a-b9b0-f2a431564e03\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.439439759Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.439963689Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.439981051Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.463189391Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.463259192Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.483848874Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.483904422Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.500690174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.500745915Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.520137653Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.52018676Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.541419987Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.541490228Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.561322542Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.561446247Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.58091783Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.580967668Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.599107433Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.599161743Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.617707453Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.617758394Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.636954275Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.637010151Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.65615917Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.656214371Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.674576675Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.674626437Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.692669633Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.692735793Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.710035265Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.710087869Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.727088419Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.727145813Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.746764326Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.749261446Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.767276777Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.767331668Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.788201532Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.788259313Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.806400764Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.806450668Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.824855159Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.824942208Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.825136892Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.825428375Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919206429Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.91929967Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.9193101Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919330201Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919340731Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919351631Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919361851Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919371971Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919381781Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919391391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919401442Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919411672Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919441092Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919463043Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919473853Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919486793Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919495163Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919504563Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.919514744Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:20.919580354Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8ce5355e-e74e-41d3-a9f6-aeafc7f19442","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized)]","latency_us":479461,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:20.91983981Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.03013743s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8ce5355e-e74e-41d3-a9f6-aeafc7f19442\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8ce5355e-e74e-41d3-a9f6-aeafc7f19442\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:23.954031974Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:23.954737782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:23.954813168Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:23.977490591Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:23.980155348Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:23.996866455Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:23.996926606Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.018761717Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.018897042Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.038940145Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.039000409Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.057801252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.05793551Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.076981234Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.077038356Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.095332809Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.097992798Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.114020122Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.116107511Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.133533771Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.13358496Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.153038759Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.153099613Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.172111107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.172162061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.190884537Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.190951428Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.209144547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.209202992Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.229630112Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.229684552Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.248664544Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.248728978Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.266583723Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.266646744Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.28630895Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.286371997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.304846018Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.304907539Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.324496505Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.324552343Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.343386197Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.343452901Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.343651017Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.34398046Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443182781Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443266455Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443277198Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443285983Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.44329338Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443302605Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443326774Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443338549Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443353221Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443384787Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443394557Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443402497Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443408319Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443413747Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.4434194Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443427022Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443434841Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443441891Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.44345302Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:24.443509523Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"cb7126d4-3fe2-4659-a594-1b42bd265003","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized)]","latency_us":488772,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:24.443749527Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.45691014s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb7126d4-3fe2-4659-a594-1b42bd265003\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb7126d4-3fe2-4659-a594-1b42bd265003\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.903501064Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.904113614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.904137754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.926970069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.927033801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.948364111Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.948439632Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.968667068Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.968734645Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.987498395Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:25.98757273Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.00840533Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.008470487Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.02800448Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.028066825Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.047781446Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.047838485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.067308811Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.067380484Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.086161414Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.086224649Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.106426398Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.106485574Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.126319187Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.126394628Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.145751129Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.14581364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.164010163Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.164087224Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.183702019Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.18376274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.202209647Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.202278948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.221461526Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.221519327Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.240724173Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.242827418Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.259526858Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.259611789Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.278766378Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.281402158Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.298598716Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.298688436Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.298948343Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.299251217Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404102699Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.40416811Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404185369Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404192382Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404198936Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404206417Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404214451Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404240223Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404256921Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404265621Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404270346Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404276646Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.40428153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404286349Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404292546Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404298921Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404304894Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404312421Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404317325Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:26.404359634Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"195bb1a3-43d8-4fc4-a8ce-fcc2a6416e73","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized)]","latency_us":500231,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:26.404669558Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.15803885s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"195bb1a3-43d8-4fc4-a8ce-fcc2a6416e73\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"195bb1a3-43d8-4fc4-a8ce-fcc2a6416e73\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.565702868Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.566314461Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.566333192Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.588260558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.59082247Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.608439242Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.608485556Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.627271547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.629857608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.646244046Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.646294934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.665126644Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.665174447Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.683709016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.683763637Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.702292381Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.702379908Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.721220189Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.7212764Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.740268789Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.740328452Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.758654589Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.758713596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.777492044Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.777542364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.796084229Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.796134911Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.814551614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.814603806Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.832982816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.833031541Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.851577901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.851624626Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.87052576Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.87057102Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.873431579Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35898"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.873751085Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"8d6e2310-faa9-4e5e-9a53-82dc2d25b77e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35898","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":139,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.888647741Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.888702902Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.906963273Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.907086495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.926431038Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.92650519Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.944952252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.945085826Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.945451306Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:29.945768523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042291414Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042380581Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042392262Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042400841Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042409456Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042417257Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042425057Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042432454Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.04244017Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042448513Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.04245636Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042464929Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042472682Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042480596Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042489127Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042494386Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042512977Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042519792Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.042524574Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:30.042584406Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"30548565-5b5a-42d3-8571-e3f78574f9eb","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized)]","latency_us":476197,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:30.04281646Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.840079214s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30548565-5b5a-42d3-8571-e3f78574f9eb\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"30548565-5b5a-42d3-8571-e3f78574f9eb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.886860141Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.8873415Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.887355356Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.911096573Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.911163333Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.928536256Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.930602571Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.947152161Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.947215574Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.968132659Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.968179093Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.987373814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:33.990305303Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.004980812Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.005036153Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.025043619Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.025108645Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.044821443Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.044875425Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.063074677Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.063131069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.082463709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.08252432Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.101593991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.101643848Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.119395901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.119451355Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.138551603Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.138620524Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.157187691Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.1572447Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.175113344Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.175185167Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.194879158Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.194937679Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.213545084Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.213602919Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.231449883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.231508637Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.250261997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.250329217Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.269650696Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.269735822Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.270362031Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.270542174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368724157Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368768932Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368775953Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368780247Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368785142Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368789454Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368793495Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368798773Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368803732Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368808082Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368812179Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368816942Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368821845Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368827836Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368843502Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368848865Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368854743Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368860902Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.368867137Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:34.368970704Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"5f40c826-45bc-4d34-9ea9-df36b4b98bf6","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized)]","latency_us":481526,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:34.369260262Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.103369171s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5f40c826-45bc-4d34-9ea9-df36b4b98bf6\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5f40c826-45bc-4d34-9ea9-df36b4b98bf6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.476969099Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.477732293Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.477747705Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.499821865Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.499871375Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.518772312Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.518830363Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.539189264Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.539242281Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.557272183Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.557323896Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.57667989Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.576751301Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.595596275Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.595655348Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.61466697Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.614722941Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.6331228Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.633240474Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.653080607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.653141628Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.67248652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.672546868Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.692355501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.692410036Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.711396814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.711460825Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.730789419Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.730849214Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.749134793Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.749191381Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.767912189Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.76796904Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.786662196Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.786723781Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.806188692Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.806238127Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.823874072Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.823934831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.843924371Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.843978213Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.86312155Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.863211007Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.863508918Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.863813283Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.960991013Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961082605Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961101035Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961126465Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961135245Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961146975Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961159986Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961168916Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961179166Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961189356Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961202986Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961222187Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961230467Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961236567Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961242367Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961250277Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961257707Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961265877Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961275757Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:38.961339589Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7011f758-dab8-4161-8d0b-add2e1c1a13a","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized)]","latency_us":483640,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:38.961600008Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"5.386677978s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7011f758-dab8-4161-8d0b-add2e1c1a13a\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7011f758-dab8-4161-8d0b-add2e1c1a13a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.353375121Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.3539325Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.35395368Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.376024382Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.376085864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.397137003Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.397188529Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.418441178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.418499846Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.438075262Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.438126704Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.458416139Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.458472081Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.47815908Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.478225992Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.496992699Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.49705015Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.516367604Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.516411395Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.534630745Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.534666943Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.554402996Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.554443385Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.573045427Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.573078774Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.591677287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.591728513Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.610218743Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.610259453Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.628640022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.628685942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.647537473Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.647585334Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.665889459Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.665927607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.68411314Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.684148766Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.702002038Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.702038376Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.720087263Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.720137739Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.738803615Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.738847087Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.73910946Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.739382342Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836479241Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836515616Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836521288Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836528751Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836535229Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836542223Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836548738Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836553839Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836561039Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836574192Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836644804Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836655501Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836659888Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836664839Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836669189Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836673454Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836677364Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836682427Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836686345Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:44.836742202Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7f8ccc31-7b34-48b3-9f96-784d99f9f59d","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized)]","latency_us":482747,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:44.836942315Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"5.31910922s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f8ccc31-7b34-48b3-9f96-784d99f9f59d\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7f8ccc31-7b34-48b3-9f96-784d99f9f59d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.161490321Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.162477282Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.162501704Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.186464964Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.186526268Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.206576631Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.206633182Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.22698787Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.227053177Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.246404881Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.249080332Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.265958174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.266021285Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.284257303Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.284310843Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.302682222Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.302740948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.321496077Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.321557887Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.341282714Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.34133873Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.359431197Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.359492138Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.378763971Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.3788339Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.399164264Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.399217618Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.418845733Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.421375389Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.438460372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.438523298Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.457952816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.458018723Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.476562958Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.478713015Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.495761048Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.495810689Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.513994636Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.514061537Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.53324625Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.535378814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.551762794Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.551836145Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.552060089Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.552213364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647214231Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647266404Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647273632Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647280279Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647285839Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647291923Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647297221Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647311021Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647316936Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647323189Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647333193Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647339652Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647345728Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647351868Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.64735739Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.64736355Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647374022Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647379197Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647385122Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:50.647461519Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3af7f18c-ebf5-4ded-9be6-3e3ee0bb2e2b","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized)]","latency_us":485371,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:50.647679323Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"5.73764568s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3af7f18c-ebf5-4ded-9be6-3e3ee0bb2e2b\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3af7f18c-ebf5-4ded-9be6-3e3ee0bb2e2b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.391184231Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.391708132Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.391728758Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.414997168Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.415061313Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.438215995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.438263442Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.459426555Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.459481371Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.479307152Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.479356483Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.500684754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.500751455Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.521676872Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.52172207Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.541167333Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.541223064Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.563234144Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.563275433Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.583121775Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.583169396Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.603253608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.603298859Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.622786558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.622829205Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.644408412Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.644454098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.66427837Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.664319499Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.68636279Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.686417311Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.706700865Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.706741356Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.727985653Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.728093828Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.748559347Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.748602304Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.768759608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.768814115Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.790242294Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.790291335Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.812335991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.812398495Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.812747494Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.812886124Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919185018Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919242157Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919252469Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919261638Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919278138Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919283622Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.91928771Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919292791Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919297207Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919300366Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919304472Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.91930767Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919310923Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919314261Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919317598Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919322164Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919326504Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919334323Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919338973Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:56.919385287Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1cb79917-b0c2-48fd-8f1b-4a68332bf856","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized)]","latency_us":527481,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:50:56.919571121Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"10.509298582s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1cb79917-b0c2-48fd-8f1b-4a68332bf856\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:50:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1cb79917-b0c2-48fd-8f1b-4a68332bf856\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:59.928288337Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:50:59.928427855Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"bbb38045-90db-4695-a8cd-368101440bd4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65493","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.437370718Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.438136834Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.438161284Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.462015935Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.462061135Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.481446707Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.481506239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.499286796Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.499320219Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.518371521Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.518417871Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.5372458Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.537281801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.555924444Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.555969405Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.574418523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.57446898Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.592470767Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.592512363Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.612417624Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.612471653Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.630629018Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.630677187Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.64957964Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.649630801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.668322084Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.668363663Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.68607144Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.686119451Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.705134573Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.705190324Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.725311367Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.725358227Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.744262296Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.744312641Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.763204406Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.763264937Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.782238458Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.782276493Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.801008983Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.801050814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.819159385Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.819204206Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.819534891Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.819709144Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.915925803Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916059422Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916083402Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916103762Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916114012Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916124223Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916136203Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916144913Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916156653Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916166903Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916176803Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916186993Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916195193Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916201244Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916223314Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916233114Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916242914Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916249304Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916257405Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2949","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:07.916299155Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d70a7577-9e0f-4e8d-b243-60abfbe8a4e4","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\", TimeNotSynchronized), (\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\", TimeNotSynchronized), (\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\", TimeNotSynchronized), (\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\", TimeNotSynchronized), (\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\", TimeNotSynchronized), (\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\", TimeNotSynchronized), (\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\", TimeNotSynchronized), (\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\", TimeNotSynchronized), (\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\", TimeNotSynchronized), (\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\", TimeNotSynchronized), (\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\", TimeNotSynchronized), (\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\", TimeNotSynchronized), (\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\", TimeNotSynchronized), (\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\", TimeNotSynchronized), (\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\", TimeNotSynchronized), (\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\", TimeNotSynchronized), (\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\", TimeNotSynchronized), (\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\", TimeNotSynchronized)]","latency_us":478186,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:07.916517578Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"16.213754369s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d70a7577-9e0f-4e8d-b243-60abfbe8a4e4\", \"content-length\": \"124\", \"date\": \"Wed, 13 Mar 2024 04:51:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d70a7577-9e0f-4e8d-b243-60abfbe8a4e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 483f2279-0a3f-444e-ac3d-24db30026ca5, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a89d5fb9-621a-48a1-8738-8924e341c714, kind: Service { id: 483f2279-0a3f-444e-ac3d-24db30026ca5 }, name: Name(\"ntp-483f2279-0a3f-444e-ac3d-24db30026ca5\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 225, 173])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e35710c2-0ad4-4310-b535-49fbc3b2e728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: f0953b77-046f-4954-9588-57eed328d77a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 16bef80b-4497-4091-b795-b5147cfeb79c, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 891ec291-a154-4e9a-b484-c05219c7ab52, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 624690b4-cee6-4070-81e8-611b39bdcdb3, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: fc1b1f72-cd47-46c1-810c-f9761356930c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 05ec9f59-db4e-4734-8295-85ee1e1f4c08, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 340ea411-015e-4310-9e09-f57902227991, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 025aaad9-fa6b-410f-a5f6-1c1c47c46cf7, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: f8c19a73-6a0d-4acc-8ac0-70d1578cd442, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 6778525f-f783-40e2-a9f8-9912dff07f97, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 8a65210f-4483-492e-bc36-138d2fd14051, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: b20d1a12-9adc-48c2-a39c-00be3468ee57, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: fb90c97d-7de4-4022-b4f1-be1904778459, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 571e2cd1-4997-4816-897c-36a5ef05150b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 12924109-8fb2-4608-85e4-921cb9c1ac43, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: a676d038-a560-4b5a-aa4b-0a36d9317152, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e3e68004-3d78-4f80-988e-5f96af4f9b02, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.135671238Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.136389459Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 615a9669-97b2-4c06-9ca1-f5cee9e02620, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.13640555Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.160105288Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 69e66ea6-9cdf-49f5-aa83-36e00f886d4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.160177079Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.181822956Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 008b60d6-d8b4-4d18-b06a-f3d5490b3230, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.181884917Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.200921423Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e35710c2-0ad4-4310-b535-49fbc3b2e728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.200976554Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.220167852Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f0953b77-046f-4954-9588-57eed328d77a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.220225083Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.23929894Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 16bef80b-4497-4091-b795-b5147cfeb79c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.239360641Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.258279215Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 891ec291-a154-4e9a-b484-c05219c7ab52, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.258338086Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.276496798Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 24d8c7e8-c65a-4393-9f69-957caf4c073f, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.27656157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.295710367Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.295772271Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.314884395Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.314948727Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.333035998Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 340ea411-015e-4310-9e09-f57902227991, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.333090689Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.352173485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c1813a-a91d-4925-a096-e92f2a0a83b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.352222676Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.371390854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3510780b-aa00-4979-b27b-0a2d8da59e0a, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.371450345Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.389704299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 8a65210f-4483-492e-bc36-138d2fd14051, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.38976871Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.408805586Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b20d1a12-9adc-48c2-a39c-00be3468ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.408861327Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.42709444Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fb90c97d-7de4-4022-b4f1-be1904778459, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.427151192Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.447690791Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 571e2cd1-4997-4816-897c-36a5ef05150b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.447762832Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.466677577Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 12924109-8fb2-4608-85e4-921cb9c1ac43, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.466728456Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.486981982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a676d038-a560-4b5a-aa4b-0a36d9317152, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.487042473Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.504990272Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3e68004-3d78-4f80-988e-5f96af4f9b02, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.505044833Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.524192461Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.524269322Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.524584207Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.524764459Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.630722498Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.633301027Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.633318908Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.68105076Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.681742411Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.681766541Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.681784301Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.681811092Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.694936396Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.697170961Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.86367286Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.888192952Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.889134506Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.925829207Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.927129657Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.927163758Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.927206938Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.927226609Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.928475683Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.929368312Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.93563278Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.941529681Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.946507739Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.952540563Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.958450524Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.96394126Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:24.969421245Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.095229781Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.099908459Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.106677589Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.111482424Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.116305049Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.120807299Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.126110282Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.131305492Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.137192024Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.143236418Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.149101479Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.154896139Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.160574168Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.166484759Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.174726098Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.182451268Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.188836087Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.195296568Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.20119597Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.207703841Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.264848079Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.307932679Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.355484974Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.392105119Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.428260591Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.431177016Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.431196296Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.497744822Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 211, 105]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.497847843Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1276","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.501658082Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.501692243Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.568546123Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 231, 154]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.568619184Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1276","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.57096791Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.57099169Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.62688203Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 252, 149]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.626940821Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1276","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.628973952Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.628993603Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.683165245Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 202, 75]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.683244106Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1276","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.685061295Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.685081215Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.74459723Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 171, 147]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.744667332Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1276","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.751805063Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.751838133Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.752837779Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.752851829Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.753259425Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.753267215Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.753428568Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.753439318Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.755568331Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.755577901Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.759758406Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.762913715Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.765435185Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.765450915Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.765458495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.76772301Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.769938725Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.769966105Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.769976735Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.771905045Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.919857366Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.926187605Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.931753601Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.938140224Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.945066808Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.950661275Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.956861942Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.962939016Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.969716732Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:25.975793796Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:29.421447222Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:29.421912439Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:29.436597088Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:29.436801141Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:36.780559012Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:36.953955729Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:37.245841018Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:37.523080349Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:37.633872233Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:37.668421688Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:37.804312043Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:37.804333814Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:38.826253367Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:38.840504058Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48184"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:38.967987231Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:38.976551024Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:38.991990245Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.0412505Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a65b55f4-0187-4361-bf64-8e2f20e59d39","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48184","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":62157,"response_code":"200"} {"msg":"Profile for oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.04572773Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.045841842Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.04831399Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.053124815Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.614618157Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.614920382Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.923952728Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:39.924346565Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:40.134177208Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:40.13435702Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 74.379301ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.660246419Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 51.933481ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.672718553Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 26.709926ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.682338313Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 58.218472ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.693444195Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 34.115724ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.717984597Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 47.341174ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.767770402Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 61.089574ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.780420813Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 54.828608ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.795673525Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 60.916827ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.810548577Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 89.429183ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.838692705Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 60.591938ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.851866859Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 64.897474ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.864513206Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 49.930974ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.878352371Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:40.879051552Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 56.417277ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.889960782Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 130.017033ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.900207401Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 134.595025ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.915998457Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 111.100373ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.928038824Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 74.501319ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.941241969Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 82.733292ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.963058579Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 148.351587ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:40.996499689Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 87.338696ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.010521267Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 126.776506ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.021715301Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 205.83749ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.032748072Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 115.073875ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.045117705Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.079299187Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 77.343435ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.093727851Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 59.981273ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.106431268Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.106486659Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 66.653785ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.142121743Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 149.637128ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.157906673Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 120.774268ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.171048373Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 267.802535ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.18370908Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 156.077232ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.197939521Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 156.117431ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.207133644Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 160.52429ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.21715496Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 282.193345ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.229835617Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 187.05387ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.239053561Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 283.03344ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.248110001Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 197.585412ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.256589743Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.256641404Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 257.93324ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.271117179Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 168.4514ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.280267012Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 146.121909ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.290750925Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 254.802597ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.304509458Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 239.036023ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.318138621Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 214.972186ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.369587241Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 313.840297ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.381980914Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 531.179527ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.394233674Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 72.410404ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.403918615Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 362.105497ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.43960803Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 335.175721ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.453577207Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 368.773661ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.464606143Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 351.39762ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.47757615Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 46.251794ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.488004052Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 389.554019ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.49811934Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 128.785173ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.507468175Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 570.708626ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.524163215Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 344.44874ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.542595321Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 467.395289ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.556543788Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 89.302008ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.565121392Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 591.863926ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.578187805Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 916.055689ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.588577907Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 637.754477ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.601549568Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.605351027Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.605414428Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.60550579Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.60552365Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.611835168Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:41.61848705Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 283.020786ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.652679007Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 279.193376ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.670138412Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 937.21345ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.726973549Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 814.380152ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.805617272Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 426.386262ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.821197534Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 35.515605ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.84067119Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 70.911702ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.867897511Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 816.583761ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.917731946Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 95.620087ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.928751977Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 410.301538ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.943020789Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 572.837195ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.956759493Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 842.269727ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.966753298Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 501.565675ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.979014999Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 37.990128ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:41.989269698Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 97.51014ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.000633535Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 452.581917ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.012636742Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 260.402977ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.034867727Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 140.470035ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.045868408Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 240.113037ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.060348524Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.32891961s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.074160059Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 568.139733ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.113922327Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 180.822425ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.131226126Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.182186862s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.190070701Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 271.00325ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.208135292Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.399149999s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.256066248Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 886.235175ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.272418792Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 822.433088ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.311362188Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 583.292519ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.378003654Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.255649951s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.418318361Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 304.902904ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.435707581Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 931.004482ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.485401244Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 1.11883532s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.500431838Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 538.083628ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.518090072Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.143527639s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.532140771Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 566.387315ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.54943137Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 595.78045ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.710280552Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 934.505569ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.740300339Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 806.7622ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.766716359Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 936.869741ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.804151232Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.470795754s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.818453714Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 1.090847532s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.830730515Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 1.138863286s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:42.976210657Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 643.768156ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.06898169Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.334435169s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.130558877Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 1.429118063s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.147472391Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.461084759s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.171438413Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 871.698136ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.318769135Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.251438674s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.386421457Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 715.712721ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.416436263Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 713.081753ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.428055764Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 1.353101358s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.592672424Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 610.421322ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.636220452Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.353138547s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.670251781Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.094154881s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.689560641Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.398021275s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.704035226Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 896.170344ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.717372044Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 764.458071ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.731102227Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.069883134s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.757140787Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 1.305310666s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:43.935338654Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 814.607331ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.128880994Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 858.032687ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.146929514Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 814.781318ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.159853655Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 877.526325ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.203743608Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 977.179635ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.258952576Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.218245918s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.303826428Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.094698716s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.482078407Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 636.298019ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.516291588Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 1.167125211s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.58905362Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 1.161984652s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.626669605Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.398179266s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.644278349Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 549.898651ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.654375346Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.494281721s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.799206739Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 535.498158ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.842227458Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 1.337569296s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.956413133Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 1.138741041s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.968939748Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 1.114514869s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:44.986768325Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 630.399762ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.017474013Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.230018471s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.037437104Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 1.182094551s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.096490422Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.368736128s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.116535234Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 540.494314ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.168195937Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 792.310834ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.219001477Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 840.175104ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.259594089Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 634.16275ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.275861102Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.306344893s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.393603493Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.065903091s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.537684424Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 590.771987ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.591234296Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.013744502s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.665185027Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 655.046809ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.72972129Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 547.169621ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.774311718Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 997.919423ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.805399627Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 574.878939ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:45.928616133Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.126753075s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.023882535Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 752.419531ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.10073913Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 831.708508ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.119518272Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 622.015107ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.131260165Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 765.740892ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.144755374Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.027542138s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.196973237Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.261590077s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.280641798Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 627.449701ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.291875803Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 662.088393ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.304857375Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 637.174378ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.314809359Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 1.153723197s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.334537186Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 648.703735ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.39907585Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 931.571373ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.498546977Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 1.23880612s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.51610095Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:46.555802337Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:51:46.557636266Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 883.942961ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.618067536Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 508.733438ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.690675935Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 563.380986ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.710669166Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 601.524226ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.766165124Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 29.701634ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.78425193Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 1.472042781s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.815236372Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 77.779377ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.82280924Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 937.324872ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.862814652Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 138.888187ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.911175824Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 1.45584726s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.922700173Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 958.709571ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:46.932562457Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 1.378327079s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.0015479Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 1.332309027s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.15014219Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.087023454s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.186465215Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 506.294731ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.197970064Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 1.348609485s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.208667931Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 866.553313ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.220534825Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.035052735s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.230720383Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.309152019s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.240959283Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 735.697176ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.287460166Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 1.378827229s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.380397101Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 676.833295ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.440449585Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 1.441797092s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.501100792Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 585.917674ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.513944538Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 749.902657ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.551561283Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 1.039352592s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.717056047Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 816.649713ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.765621273Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 854.593577ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.811511536Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 1.006773475s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:47.903290654Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.050832329s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.034544875Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.411137622s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.099762577Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 500.216791ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.110478206Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 575.218474ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.131192748Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 659.333984ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.275522893Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 990.2635ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.2830619Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 1.106729831s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.2959072Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 608.358538ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.310289903Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 962.50402ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.388053913Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 670.904845ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.395836024Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 1.236207008s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.491700485Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.217448232s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.560234071Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 1.120918204s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.56729132Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 681.07885ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.591492497Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.444831167s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.61868832Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.187619394s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.673816807Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.16333144s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.714937226Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 554.15582ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.765374631Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 1.091420031s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.772286509Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 680.006156ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.919102362Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 573.602032ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.928089812Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.288450416s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.95178551Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 1.359442331s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:48.966489789Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 1.433381273s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.076833025Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.496219639s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.095566336Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 1.397082622s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.282495693Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.034562329s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.291700776Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 509.114719ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.330167094Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 571.010322ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.360286033Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 621.818913ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.412083728Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 694.082766ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.512084894Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 661.968987ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.520373243Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 926.53881ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.608513814Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 1.033457014s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.698752767Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 890.406582ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.739488911Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 830.500907ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.787749451Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 505.277438ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.8507216Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.11107356s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.875172971Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 975.849962ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.888388186Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.338170295s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.897988025Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 676.101349ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:49.940662309Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 535.531282ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.045307557Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.270963845s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.07638061Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 810.180123ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.197560524Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 747.798286ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.220324548Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.048605698s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.261309856Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 607.91313ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.347962823Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.488323998s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.369116852Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 1.345297852s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.395397361Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 994.245161ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.535291947Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 1.472364672s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.553075043Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 811.659449ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.599870731Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.493768417s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.616599111Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 1.199045165s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.635069739Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.381218535s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.654232257Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 830.312643ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.673554707Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 992.430106ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.698872826Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 1.34873347s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.755808186Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 583.881237ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.883596074Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 1.02037341s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.974833383Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.384108284s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:50.990933533Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.009062605s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.005680523Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.212225282s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.025312088Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 637.613332ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.252212156Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 860.815396ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.322972487Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.274152948s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.361976243Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 671.366919ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.426171802Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 1.055956676s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.482086131Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 1.462639852s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.517844587Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 720.814375ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.542360179Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 1.1243147s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.706439761Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 579.695885ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.758519831Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 1.106175773s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.848736894Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.2849178s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.875607672Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 872.626437ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:51.905812111Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 776.764632ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.018611296Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.196791616s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.03495347Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 670.487948ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.048163575Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 733.396117ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.06069714Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 905.014768ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.114360894Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 533.091446ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.126181868Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.234973585s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.138076023Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 797.482101ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.197012Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 798.955499ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.250045365Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 751.188044ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.276199071Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 1.080903487s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.350442726Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 673.729331ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.385810346Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 1.129333599s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.548606028Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 739.728687ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.645321062Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 519.98116ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.666598319Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 540.202634ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.728577927Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.047448752s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.790957697Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 900.04082ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.804302155Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 976.30321ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.813751672Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 1.37426461s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.840742101Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 876.498097ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.965206077Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 848.84664ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:52.99086799Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.189803733s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.005332531Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 679.398956ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.029756531Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 963.192195ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.039847238Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.472121879s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.059003142Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.155189451s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.069930996Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.346995721s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.170268886Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 578.736133ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.196678873Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.167576122s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.241717837Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 1.120263881s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.278113913Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.023529888s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.3833736Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.234526189s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.394367871Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 606.920563ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.440357047Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 661.859535ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.688813671Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.289733251s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.716277588Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 1.36861824s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.724971543Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 1.067758987s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.785493404Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 572.397197ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.798035969Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 526.586724ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.848062977Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 918.93002ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.856792263Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 1.496437172s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:53.865454768Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 1.045219208s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.015025394Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 1.008097156s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.056885895Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 670.592986ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.205675116Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 845.956982ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.225448506Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.015931132s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.23599462Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 639.846219ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.361476262Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 666.190498ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.380682051Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 859.652707ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.389595839Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 1.461596114s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.40955188Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.45004692s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.418661141Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.375418576s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.427116047Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.222671796s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.527825489Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.314303005s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.540798781Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.348069379s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.640082655Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 1.046729375s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.785919233Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 1.173450893s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.86288985Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.352823043s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:54.887052166Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 1.130755437s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.012229842Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.378314938s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.021799001Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 695.65966ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.058044715Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 1.354486948s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.069864899Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 1.001988544s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.078405032Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 842.662397ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.085675775Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 1.065638055s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.10402785Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.114312719s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.258293149Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 529.723592ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.266835692Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 757.546583ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.371421739Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 897.786009ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.761331238Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 803.14863ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.769974507Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 1.247478909s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.807868276Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 1.144785359s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.816516581Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 1.450325351s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.842505375Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 885.63263ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.864645479Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 1.185099384s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.878543366Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 1.200520124s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.886885755Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 758.214865ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.937695445Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 559.074056ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:55.997703369Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c\") failed: Property not found. retry in 1.336604805s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.045858097Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 798.19071ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.089642758Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57\") failed: Property not found. retry in 1.201010716s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.138289745Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 941.441698ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.153204257Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152\") failed: Property not found. retry in 1.189313276s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.179308043Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 891.570675ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.248691242Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442\") failed: Property not found. retry in 1.267197102s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.382389921Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 1.215067703s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.407963609Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 587.258819ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.433961543Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459\") failed: Property not found. retry in 1.191493691s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.566960782Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3\") failed: Property not found. retry in 1.003131908s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.582880869Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051\") failed: Property not found. retry in 1.007260529s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.669232892Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7\") failed: Property not found. retry in 816.245747ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.706354229Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a\") failed: Property not found. retry in 1.476970567s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.759703689Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 1.349565232s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.897956389Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97\") failed: Property not found. retry in 906.129085ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:56.970377326Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6\") failed: Property not found. retry in 963.181117ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:57.032974329Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b\") failed: Property not found. retry in 846.468966ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:57.065055448Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\") failed: Property not found. retry in 956.198856ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:57.07352337Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08\") failed: Property not found. retry in 1.389195827s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:57.095851557Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02\") failed: Property not found. retry in 868.022565ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:57.103722829Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825\") failed: Property not found. retry in 1.444501366s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:57.148982644Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3\") failed: Property not found. retry in 570.427895ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:57.303263663Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43\") failed: Property not found. retry in 867.327897ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:58.014173829Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 888.968075ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:58.346333285Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991\") failed: Property not found. retry in 1.495978855s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:51:59.246116168Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.78040056Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_3510780b-aa00-4979-b27b-0a2d8da59e0a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780471831Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_pantry_f8c19a73-6a0d-4acc-8ac0-70d1578cd442"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780483881Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_nexus_d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780490461Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_fb90c97d-7de4-4022-b4f1-be1904778459"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780499381Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_571e2cd1-4997-4816-897c-36a5ef05150b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780505171Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_a676d038-a560-4b5a-aa4b-0a36d9317152"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780520391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_e3e68004-3d78-4f80-988e-5f96af4f9b02"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780529082Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_nexus_624690b4-cee6-4070-81e8-611b39bdcdb3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780544162Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_b20d1a12-9adc-48c2-a39c-00be3468ee57"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780553652Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_pantry_6778525f-f783-40e2-a9f8-9912dff07f97"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780559112Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_external_dns_62e81a8e-9f5e-4f8b-ab77-1b6098b89825"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780564512Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_8a65210f-4483-492e-bc36-138d2fd14051"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780570592Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_external_dns_eed3f3eb-69ed-4a60-b845-9b5af9f488d3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780576412Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_clickhouse_340ea411-015e-4310-9e09-f57902227991"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780582103Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_12924109-8fb2-4608-85e4-921cb9c1ac43"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780587793Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_pantry_025aaad9-fa6b-410f-a5f6-1c1c47c46cf7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780596013Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_crucible_36c1813a-a91d-4925-a096-e92f2a0a83b6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780659734Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_nexus_fc1b1f72-cd47-46c1-810c-f9761356930c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.780711234Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2945","zone":"oxz_oximeter_05ec9f59-db4e-4734-8295-85ee1e1f4c08"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.781834982Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.79390703Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.804895471Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"73290cbc-4e62-40b4-827f-8817c056eff5","remote_addr":"[fd00:1122:3344:101::1]:58772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":36666752,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.827768697Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.828283925Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.828391916Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.831737558Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.831748828Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:00.908594159Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:03.435700955Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:03.550618872Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:03.650275332Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:03.809932205Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:04.012562207Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:04.303341629Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:04.557280658Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:04.993272639Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:05.197587926Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:05.758799844Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:06.413212922Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:07.339277384Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:08.024332548Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:08.767519296Z","hostname":"sock","pid":1671,"sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1138","retry_after":"23.012042451s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:08.898863569Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"214.026623486s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:08.925009695Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"214.054057392s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:08.939579682Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:09.077173442Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38473"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:09.077452576Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"0241be48-75b3-4474-96cd-904a04808c98","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38473","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.168605931Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:52:09.175407689Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.180126793Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.183400804Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.185966104Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.268542418Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.274813355Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.276990729Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.280438833Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.292733494Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"214.431014515s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.293243802Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.296407801Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.299167164Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.301597192Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.303822577Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.306019051Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.308801764Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.311139251Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.31307271Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.316510524Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.318447314Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.321280098Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.323283299Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.326179364Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.328602042Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.331141911Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.333721802Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.336113669Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.338652818Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.341210458Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.343753098Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.346923627Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.349521927Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.352188049Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.354239711Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"214.49307352s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.354279321Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.356408246Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.785784842Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"214.924451769s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.84482168Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"214.982444931s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.848973655Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.183305925s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.85890941Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"214.996345007s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.907480575Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.046282724s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:09.921723847Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.060647877s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.050120513Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.187094664s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:52:10.178606591Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.239000051Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.378171566s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.263817116Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.401803893s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.304273846Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.442388285s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.388503856Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.52634292s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.39003892Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.52894519s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.488173186Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.490469201Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.627872089s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.490939459Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.49356045Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.495940247Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.498299863Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.500096081Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.502822204Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:10.505102679Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.505323203Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.507342794Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.509082741Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.51158347Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.514364063Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.516871342Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.519068066Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.521488654Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.523848141Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.52635626Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.528902309Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.530851129Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.533230586Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.536129641Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.538943366Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.541277392Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.544255738Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.546849128Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.54953023Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.788170111Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.925879924s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.808172013Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.945429248s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.851269863Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.990148534s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.857835115Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.994784206s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.868185926Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.006378447s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:10.878637745Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.016058197s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.11706581Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.255791995s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:52:11.182072567Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.196131066Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.333944911s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.291925046Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.429978354s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.30553851Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.442806334s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.356066203Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.493743606s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.642722392Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.780510136s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.674456545Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.811790093s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.678223224Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.680948596Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.683470985Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.685952804Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.688585475Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.691571181Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.694148081Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.696848373Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.699436543Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.702472971Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.704881008Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.708050408Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.710300643Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.71269384Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.715178968Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.71783681Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.720355869Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.722679695Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.725804184Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.728221961Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.866483693s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.728660748Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.73135484Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.734019333Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.827650017Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.965670326s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.828198426Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.967502935s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.847186972Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.984834374s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.88823441Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.026131437s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:11.955960365Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.092900935s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.000371834Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"217.138499235s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.064173616Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"217.202846865s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.17374051Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"217.312539061s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:52:12.186219054Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.242725173Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.380477887s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.271107254Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.40828466s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:12.282281388Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.431118103Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"217.569336114s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.557182683Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"217.696442802s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.593057991Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.730662134s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.67208522Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.808926801s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.69200187Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.829853957s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.700510842Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.83777977s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.82831633Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.16759434s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.838282225Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"217.975769966s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.868250471Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.870988684Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.873370611Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.876071823Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.878258718Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.216532341s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.878859468Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.882086306Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.88492382Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.887191816Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.889666914Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.892445777Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.894953196Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.897435385Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.900054846Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.903231205Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.905791145Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.908313054Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.047204547s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.908979374Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.912814294Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.91641057Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.919863444Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.923277407Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.925767306Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.927942189Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.930566976Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.933339643Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:12.93631337Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:13.038311156Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.17660118s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:13.17942121Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.316739149s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:52:13.189450296Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:13.18970511Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.527808472s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:13.896688375Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.234885349s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.038356129Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.176782836s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.068365796Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.071054937Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.074309238Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.077109421Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.079438258Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.081955797Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.084413815Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.086830753Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.089793199Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.092417564Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.09503581Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.098312401Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.100740068Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.103831597Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.106488718Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.109434054Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.111902032Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.11432845Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.116839559Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.119694283Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.122394085Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.125476203Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:52:14.192716919Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.248392685Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.386152592s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.258401911Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.396518012s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.36833446Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.70752455s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.398341117Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.5371538s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.408315632Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.545372318s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.418372208Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.556358959s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.428365614Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.566240593s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/db9a57fa-f70a-4fe0-9bbc-f139e390d48e/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:14.442721627Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.488396168Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.627541966s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.558399016Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.696929245s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.568393582Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.707058732s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.628460956Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.765699285s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.648397426Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"219.78659156s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.658598235Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.795422357s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.718442855Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.056453587s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.803474648Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.941196034s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.903118167Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.242264957s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.907824871Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.245972614s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.938408126Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.076004401s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:14.938463027Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.07593401s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.055319634Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.194432073s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.116183031Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.254544108s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.118327121Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.257076837s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.182134697Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.520109738s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-03-13T04:52:15.196377458Z","hostname":"sock","pid":1671,"component":"ProbeManager","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.207240877Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.345404301s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.258453394Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.261653233Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.26530456Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.267940501Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.270002753Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.273106154Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.276636016Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.279769705Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.282852223Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.286419808Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.288801156Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.291789492Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.296091779Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.299047125Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.30322192Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.306007393Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.308943039Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:618"} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.373803967Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.510689921s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f3f4ca39-deb2-4c5a-b1d9-d5be0cd2aaba\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"f3f4ca39-deb2-4c5a-b1d9-d5be0cd2aaba\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.37613717Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.512865775s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f2066840-cff4-4b62-a47b-d7d03523f8d4\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"f2066840-cff4-4b62-a47b-d7d03523f8d4\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.378690494Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.515796381s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"671b2e11-00a7-4d90-bb78-8d1877b45465\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"671b2e11-00a7-4d90-bb78-8d1877b45465\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.38041234Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.517736771s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"a9ac6cd1-5a68-4069-8c8f-b169d7347e8a\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"a9ac6cd1-5a68-4069-8c8f-b169d7347e8a\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.392249614Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.530286116s","err":"\"Error Response: status: 403 Forbidden; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0a6534bf-7e04-4fda-b373-f01b454146f4\\\", \\\"content-length\\\": \\\"113\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"Forbidden\\\"), message: \\\"Forbidden\\\", request_id: \\\"0a6534bf-7e04-4fda-b373-f01b454146f4\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.401711301Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.53890266s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"d5c94c96-7b74-4cf9-9115-e0022ebe6658\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"d5c94c96-7b74-4cf9-9115-e0022ebe6658\\\" }\""} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:15.468690303Z","hostname":"sock","pid":1671,"component":"NexusNotifierTask","sled_id":"d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","component":"SledAgent","file":"sled-agent/src/nexus.rs:591","new_generation":"1","old_generation":"1"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:15.554540258Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.764567195Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.901297897s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"a9cf0a9d-29ad-426f-8d71-7f0ef8855fe2\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"a9cf0a9d-29ad-426f-8d71-7f0ef8855fe2\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.767188876Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.904054299s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"137fba2a-20a0-4451-bdfc-bc8ce2fdd697\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"137fba2a-20a0-4451-bdfc-bc8ce2fdd697\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:15.838997262Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.975823106s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"a4327f8b-b31a-4301-af7c-3006e51f2446\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"a4327f8b-b31a-4301-af7c-3006e51f2446\\\" }\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.0503619Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.052070586Z","hostname":"sock","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"4bd07e61-f02e-42d0-828b-6b7cf129fe15","remote_addr":"[fd00:1122:3344:101::b]:53539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1549,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.052178848Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45221"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:16.05230191Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.189820624s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7a695343-7e38-4ffc-8cd0-2d01abccf989\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"7a695343-7e38-4ffc-8cd0-2d01abccf989\\\" }\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.053095992Z","hostname":"sock","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"1af432b8-8d61-4a8a-b416-7834d0c9554a","remote_addr":"[fd00:1122:3344:101::c]:45221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":773,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.095453851Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36229"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.096296594Z","hostname":"sock","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"89474d01-cbb9-405a-8862-877cda9eaa2b","remote_addr":"[fd00:1122:3344:101::a]:36229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":781,"response_code":"200"} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:16.103033208Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.240318109s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"67ed23e9-e219-464e-bbea-ff774ae7cac9\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"67ed23e9-e219-464e-bbea-ff774ae7cac9\\\" }\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.118943166Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.121599267Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"3c68c649-bd79-4a92-9530-17bb5da076d6","remote_addr":"[fd00:1122:3344:101::b]:48738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2597,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.121646418Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:55876"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.121674938Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"bb2a780f-2d10-42f3-af56-47552edca090","remote_addr":"[fd00:1122:3344:101::c]:55876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":15,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.122280028Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.122300128Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.124437441Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.124654465Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"a00b2a4c-df51-40bb-a53a-4aa6727cc94b","remote_addr":"[fd00:1122:3344:101::b]:48738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2843,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.124667645Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.124686995Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.125183873Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.125384616Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"8d91eb81-f5df-443b-aaa3-1a5b6781409c","remote_addr":"[fd00:1122:3344:101::c]:55876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2867,"response_code":"200"} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:16.136670162Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.27381572s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"5993b17d-eb60-4868-badc-bef4e55639c4\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"5993b17d-eb60-4868-badc-bef4e55639c4\\\" }\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.140888117Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.140980419Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"1917d47e-da57-4574-aa37-bcc7e6b55bbc","remote_addr":"[fd00:1122:3344:101::a]:60467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.141242553Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.141257283Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.141468516Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.1416857Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"5842beda-5df3-45b3-9444-7daa40e4290c","remote_addr":"[fd00:1122:3344:101::a]:60467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":466,"response_code":"200"} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:16.261809708Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.399421174s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"84c05c01-358c-4dab-898e-1a4a039d1572\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:15 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"84c05c01-358c-4dab-898e-1a4a039d1572\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.360118787Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:52:16.46253491Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.599233562s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"1ffd8c8f-55d0-44e1-84e8-5d1e060c6fbc\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Wed, 13 Mar 2024 04:52:16 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"1ffd8c8f-55d0-44e1-84e8-5d1e060c6fbc\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.552798523Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.552818044Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.552827804Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.552843214Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.552848014Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.552852384Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.552856554Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.640900558Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.640915502Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:16.640921155Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.230816478Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.230853839Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.230878439Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.230883489Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10c4dd0b-4b64-40c3-8819-20867dfb9c1b\", \"content-length\": \"174\", \"date\": \"Wed, 13 Mar 2024 04:52:17 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\"\", request_id: \"10c4dd0b-4b64-40c3-8819-20867dfb9c1b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.629025741Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.655261379Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.655288829Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.726542068Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:17.726554588Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:18.077812161Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:18.077835402Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.915086966Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62304"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919419414Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919601836Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919646557Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919664028Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919670217Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919687638Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919695368Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:20.919724658Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"617efe16-ef70-44f8-9e73-d00a83dc8562","remote_addr":"[fd00:1122:3344:101::a]:62304","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3970,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12fab566-c104-41d6-bb9c-d64d3cab2931\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:52:21 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"12fab566-c104-41d6-bb9c-d64d3cab2931\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:21.052061567Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:21.120151175Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60541"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:21.120223786Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"0546ede6-8fea-4245-a501-872d3defa73b","remote_addr":"[fd00:1122:3344:101::a]:60541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:21.120400499Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:21.12043933Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:21.120703705Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:21.120972546Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"485ba13e-fbc7-4554-9830-ff463827ca45","remote_addr":"[fd00:1122:3344:101::a]:60541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":641,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553005131Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58765"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553262665Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553306446Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553330576Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553350407Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553381517Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553390467Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553397077Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.553407897Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0143c306-b18c-4f9d-bd98-a5b1f2bc3e2f","remote_addr":"[fd00:1122:3344:101::a]:58765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":319,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"19a7cf06-5a93-4237-94dd-328157312660\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:52:23 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"19a7cf06-5a93-4237-94dd-328157312660\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.659606529Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.795151517Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55102"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.795245359Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"dd661785-d96c-4eaa-80b8-3a8ce3fb66b4","remote_addr":"[fd00:1122:3344:101::a]:55102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":55,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.795473902Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.795498203Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.795799817Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:23.796245914Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1e4d7188-ae57-4bcc-8429-44a3bc0da478","remote_addr":"[fd00:1122:3344:101::a]:55102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":821,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.491703086Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46196"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492021251Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492092182Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492143183Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492171033Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492180464Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492194484Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492229674Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.492265465Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"243acb87-376b-4911-bbf5-786fb120adaf","remote_addr":"[fd00:1122:3344:101::a]:46196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":494,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5dd2eea9-20ff-40ed-bcba-45ad292bc9da\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:52:27 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5dd2eea9-20ff-40ed-bcba-45ad292bc9da\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.556982561Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.704729419Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.705448116Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"2f282dc2-3aeb-4f41-a721-90a335666a4b","remote_addr":"[fd00:1122:3344:101::a]:63164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":694,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.705604991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.7056281Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.705887135Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:27.706185632Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"dec491bd-7019-4fb0-a1d3-2c74c6e4319b","remote_addr":"[fd00:1122:3344:101::a]:63164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":585,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974658623Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39204"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974861946Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974899887Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974922137Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974931117Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974943017Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974949928Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974960858Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:32.974984398Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8565e696-e61b-49fd-bdb4-3a615662a546","remote_addr":"[fd00:1122:3344:101::a]:39204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":292,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d1cbb658-1b4b-457f-8cac-0ded0a208a5d\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:52:32 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"d1cbb658-1b4b-457f-8cac-0ded0a208a5d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:33.035103193Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:33.15388056Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:33.153997972Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"088f3a7a-cc47-4898-8301-1dbfe2a7214c","remote_addr":"[fd00:1122:3344:101::a]:61828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:33.154255126Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:33.154298447Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:33.154575351Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:33.154829015Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"7bf245c6-5deb-4567-ac00-1aaf95ece44a","remote_addr":"[fd00:1122:3344:101::a]:61828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":632,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.108656224Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54255"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.108758566Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"f6f76c3e-ebf0-4fe3-8c2a-404a7b091320","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54255","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.849554308Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56746"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.850123706Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.850184297Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.850247988Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.850277099Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.850310819Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.85035396Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.85038943Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.850415591Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0d678d80-9ac8-4ff5-a177-1b89f3684df4","remote_addr":"[fd00:1122:3344:101::a]:56746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":546,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fd89208e-c0d2-4a39-a905-690d243c537a\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:52:39 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"fd89208e-c0d2-4a39-a905-690d243c537a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:39.905941605Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:40.007031657Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:40.007124938Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"1aa3c561-e38c-4fdb-a617-61e35c501fd3","remote_addr":"[fd00:1122:3344:101::a]:61823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":63,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:40.007922971Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:40.007968311Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:40.008274776Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:40.008475529Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"c02b8451-06a1-4bd5-b6ce-b52e3fb6e2f9","remote_addr":"[fd00:1122:3344:101::a]:61823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1223,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893346019Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47295"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893604752Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893649243Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893698284Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893710234Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893720364Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893728334Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893740495Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.893768285Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"284ee4cb-754a-43c9-9fe6-3f8d76c04f2c","remote_addr":"[fd00:1122:3344:101::a]:47295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":377,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dee36dad-498b-41a9-989b-ea90f8e27fa5\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:52:46 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"dee36dad-498b-41a9-989b-ea90f8e27fa5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:46.946332243Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:47.061329061Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:47.061416442Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"53370a29-83de-4f89-99ff-295049c11126","remote_addr":"[fd00:1122:3344:101::a]:48173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":59,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:47.062236645Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:47.062261776Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:47.06253478Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:47.062784634Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"f51cc219-fdb0-4b15-ae7b-158f913c8d7a","remote_addr":"[fd00:1122:3344:101::a]:48173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1230,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975448548Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:45739"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975696732Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975750583Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975775124Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975790974Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975813594Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975826084Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975836465Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:54.975852605Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2d0285b9-0dbd-488e-bebf-665628df60ac","remote_addr":"[fd00:1122:3344:101::a]:45739","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":368,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e658fcd-829d-4580-a8e2-b5ca1882afd7\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:52:54 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5e658fcd-829d-4580-a8e2-b5ca1882afd7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:55.045252794Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:55.152690715Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62350"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:55.152791866Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"a2f6eea1-8700-48e1-b1ea-625e2f0f7a88","remote_addr":"[fd00:1122:3344:101::a]:62350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":68,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:55.153594619Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:55.15363774Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:55.153922404Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:52:55.154180838Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"456a014f-5d06-4d70-a54d-646ff95006f7","remote_addr":"[fd00:1122:3344:101::a]:62350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1209,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.035712823Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55993"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.035978547Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.036100179Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.036124909Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.036137269Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.03614888Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.03616543Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.03617649Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.03620141Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"5e529e62-fa1a-486d-953a-b6de9ca64573","remote_addr":"[fd00:1122:3344:101::a]:55993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":424,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e2cc118-a9c1-49a0-b513-bb1e2047b689\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:53:01 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5e2cc118-a9c1-49a0-b513-bb1e2047b689\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.098618181Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.23549907Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.23613338Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"74442d79-85ad-4f04-8756-8b40bf16c0c7","remote_addr":"[fd00:1122:3344:101::a]:48226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":623,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.236695709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.236725449Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.237027584Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.237300248Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"619265c6-cb9d-4fd1-a70c-f7b2ba83d417","remote_addr":"[fd00:1122:3344:101::a]:48226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":661,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.862704275Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:62491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:01.872106691Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"99a45a80-af5f-4af7-9f81-c9f9303dd4c7","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9351,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:09.140296212Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:09.1407987Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"7d539208-29b7-4f64-927b-e19229772baa","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45807","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":435,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.422817355Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57184"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.423077149Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.42313693Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.423180081Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.423196441Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.423208271Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.423221782Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.423237222Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.423266302Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1d5be508-ccc7-4dd4-9252-1fde88860f64","remote_addr":"[fd00:1122:3344:101::a]:57184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":393,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0460869-3bfa-49aa-8ba2-7705888723d0\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:53:11 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"d0460869-3bfa-49aa-8ba2-7705888723d0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.477674749Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.579513222Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53978"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.579603794Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"378b6b1e-bcc0-4dae-a3f9-47a88c2c6551","remote_addr":"[fd00:1122:3344:101::a]:53978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":59,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.579805107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.579847827Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.580291165Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:12.580580839Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"de615415-b14b-477a-911e-d666c8caae97","remote_addr":"[fd00:1122:3344:101::a]:53978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":830,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.714991145Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59149"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.715228189Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.71528239Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.71531508Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.715327731Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.715337851Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.715346111Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.715366641Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.715385181Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c2d61c89-ff7b-4bb0-bea8-3fd6f0450f17","remote_addr":"[fd00:1122:3344:101::a]:59149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":357,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a118de58-fed2-4b8d-aa31-6745f0602435\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:53:27 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a118de58-fed2-4b8d-aa31-6745f0602435\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.779666781Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.885623039Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:49287"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.885714161Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"97ae7e2f-282b-46e3-95fc-b7685aa69639","remote_addr":"[fd00:1122:3344:101::a]:49287","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":64,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.885884373Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.885916634Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.886256959Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:28.886464882Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"e8febb38-bdbb-4c5a-9d85-0f1c80ef37fe","remote_addr":"[fd00:1122:3344:101::a]:49287","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":632,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:31.862764502Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"6cc60d58-7f72-43ae-8923-07a6df72677f","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":502,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.17499701Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39994"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.175117142Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"1cd548b4-241a-4e29-8f23-47ee5ccfc23a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39994","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302122757Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65162"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302342821Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302389661Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302416402Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302444042Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302455482Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302475853Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302493623Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.302516953Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6faa973b-b893-4f35-8e40-5cb8785c9e98","remote_addr":"[fd00:1122:3344:101::a]:65162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":349,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ed7291c9-f68e-4037-b048-e943a081ec74\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:53:38 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ed7291c9-f68e-4037-b048-e943a081ec74\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.354525672Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.45083419Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.450904851Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"8b03246a-f66b-4bd4-bef1-67e1e9c38b66","remote_addr":"[fd00:1122:3344:101::a]:41799","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.451064974Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.451095694Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.45145153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:53:39.451689394Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"7d02370f-6c03-4157-852c-8f44f2fc4d10","remote_addr":"[fd00:1122:3344:101::a]:41799","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":667,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163287306Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40061"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163650571Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163716472Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163754983Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163767983Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163779883Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163815094Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163842244Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.163883875Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"51c33f27-865a-46f3-bbd3-34887c4ad9de","remote_addr":"[fd00:1122:3344:101::a]:40061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":497,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"afeba17d-5d2e-4b60-a80a-8c226a17691c\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:54:00 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"afeba17d-5d2e-4b60-a80a-8c226a17691c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.225379671Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.328987543Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.329079734Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"a1a64069-6768-4bb9-947f-bd6e3175a963","remote_addr":"[fd00:1122:3344:101::a]:53897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":59,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.329299778Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.329341028Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.329720424Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.330002209Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"672ca78f-714a-4d27-90f6-39a8a569f4a8","remote_addr":"[fd00:1122:3344:101::a]:53897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":759,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:01.86309799Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"b264b2bc-af46-4c70-bd69-292d5626ef5d","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:09.206499182Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:32923"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:09.206612283Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"787efb95-e7bf-4de2-b9e4-0f70c8f73f18","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:32923","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:17.964111528Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.697974902Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62989"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.698831465Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.698893336Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.698936567Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.698966547Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.698989348Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.699009478Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.699025098Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.699051639Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"13a6c72b-e3ad-4b9e-bd95-679c6ba3d329","remote_addr":"[fd00:1122:3344:101::a]:62989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":978,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4a494035-2b63-4bae-82f2-87c0b5e27cd6\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:54:18 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"4a494035-2b63-4bae-82f2-87c0b5e27cd6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.754899517Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.850318611Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57635"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.850394643Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"1ab7d124-cbb3-4eda-a99f-c50533092aa7","remote_addr":"[fd00:1122:3344:101::a]:57635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.851168775Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.851200285Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.85152098Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:18.851742813Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"43c45050-74ed-433e-b946-1151b72fabcb","remote_addr":"[fd00:1122:3344:101::a]:57635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1222,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:31.863508055Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"b400b569-b6ba-4da7-aee8-4d3263bea0f6","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1023,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.89894758Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60052"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899256645Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899320786Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899339046Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899362997Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899394747Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899407137Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899427138Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.899451288Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b6fa8fcb-7ad1-4cab-84c7-d6629149c909","remote_addr":"[fd00:1122:3344:101::a]:60052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":456,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4e3e7880-8954-40be-ba89-f1aa225b060e\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:54:36 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"4e3e7880-8954-40be-ba89-f1aa225b060e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:36.95552491Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:37.047323178Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:37.047434489Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"18b9c5a3-2d71-4132-b077-8ae05ea35802","remote_addr":"[fd00:1122:3344:101::a]:61640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":79,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:37.048285873Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:37.048313603Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:37.048674009Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:37.048940213Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"373d4318-60c6-467c-85a1-2201818391ff","remote_addr":"[fd00:1122:3344:101::a]:61640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1336,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:39.242913136Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:39.243020677Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"b168e8ab-4bee-487c-9152-25c39b4f1311","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58048","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.001690584Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62381"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.001937037Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.001991458Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.002022559Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.002035649Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.002047859Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.002066389Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.00208754Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.0021162Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"63a28a99-d12e-48ad-aaf9-daf9234f1e35","remote_addr":"[fd00:1122:3344:101::a]:62381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":362,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c048b827-8f85-4bd6-89ba-00dfd1f60701\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:54:52 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"c048b827-8f85-4bd6-89ba-00dfd1f60701\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.061119798Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.217063293Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46086"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.217148794Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"0fc665fe-45a1-48dd-841e-65660fd742fe","remote_addr":"[fd00:1122:3344:101::a]:46086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":62,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.217345727Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.217373408Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.217680732Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:54:52.217922026Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"367d92d4-d7ea-43e9-a9bb-e6b60e766b61","remote_addr":"[fd00:1122:3344:101::a]:46086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":632,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:01.863513205Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"b0b22311-f8ff-4af2-aa60-2a4c18e0b8a2","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":823,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:09.275144278Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:09.275253869Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"ed32f510-2953-4979-baae-44fce8c8dfab","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55650","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:31.863581625Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"8f3b091a-6b2d-4845-8b12-9c25d0a6256e","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1254,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.78813349Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43107"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788465586Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788530827Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788557627Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788568607Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788574207Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788592768Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788605108Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.788631628Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"25632920-8244-427d-8c29-b87defbbee14","remote_addr":"[fd00:1122:3344:101::a]:43107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":435,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2433765f-2329-4244-98f8-f6f4b90ad996\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:55:34 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2433765f-2329-4244-98f8-f6f4b90ad996\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.850333858Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.952438936Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37001"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.952539187Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"c874e361-0d6b-4d67-92a7-e1f1a5c2f23a","remote_addr":"[fd00:1122:3344:101::a]:37001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":70,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.952826502Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.952906483Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.953562133Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:34.953872978Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"32afa1f8-60cd-4a1d-b200-af4b422cdd35","remote_addr":"[fd00:1122:3344:101::a]:37001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:39.307939997Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:39.308049198Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"7d3b5756-53c5-4954-b092-8a7b689ed391","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45048","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.908502317Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:32840"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.909286849Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.90934193Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.909378611Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.909397171Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.909406551Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.909418671Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.909432412Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.909459122Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f15a3361-73c9-4604-9f6d-af950f5feaad","remote_addr":"[fd00:1122:3344:101::a]:32840","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":893,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4fd4bf01-5129-4701-8002-2f1ff9ae1e77\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:55:56 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"4fd4bf01-5129-4701-8002-2f1ff9ae1e77\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:56.964110042Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:57.05527592Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46411"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:57.055342571Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"db33aa00-a2bc-45d8-8ebc-45585e5f5fab","remote_addr":"[fd00:1122:3344:101::a]:46411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:57.055498593Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:57.055525964Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:57.05589229Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:55:57.056185614Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"88543ade-0ee9-492a-81db-8bcad1d13f68","remote_addr":"[fd00:1122:3344:101::a]:46411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":740,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:01.8633561Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"18035736-ab44-43bf-841a-f5693e4fbadb","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":503,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:09.339560167Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49168"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:09.339659268Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"ff37de6c-8f6a-4ad8-8982-584ad8295315","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49168","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":74,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:31.863284457Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"fff57c16-4b98-40bc-938c-d88b6f654eb3","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":585,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523562292Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47149"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523758695Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523790405Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523828746Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523836246Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523842876Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523855646Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523873967Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.523894067Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f151cd98-2274-4069-9cc0-1f970d1f3d8c","remote_addr":"[fd00:1122:3344:101::a]:47149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":308,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"107427f5-ca90-42fc-a4de-655b14dc35ae\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:56:34 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"107427f5-ca90-42fc-a4de-655b14dc35ae\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.644027615Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.74595151Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.746053542Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"5b146171-3126-4d9f-9d4d-aab519483637","remote_addr":"[fd00:1122:3344:101::a]:64915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":74,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.746250985Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.746289096Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.746689962Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:34.746982357Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"3c8e434f-2cba-472c-a05f-9c7e56c80109","remote_addr":"[fd00:1122:3344:101::a]:64915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":802,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:39.373191158Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49952"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:56:39.373760907Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"339f08dc-748e-49ca-bd46-e6451d7c4ebd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49952","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:01.86349791Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"d0c0eedc-32cd-4052-85bf-a86a20f02a5b","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:09.406508247Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43782"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:09.406589118Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"70e985cb-9499-49ca-9a7c-29b19e98ca6b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43782","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":72,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:31.86353986Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"785c3366-c66c-4ada-ad34-3f3ca20db0a2","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":650,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:39.440080228Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54908"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:39.440183789Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5f5b5d50-7350-401d-89e4-0ed098c4f367","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54908","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731521816Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55655"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731792131Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea00bda-e10d-4aaf-b080-27144ee341c5, kind: Service { id: eed3f3eb-69ed-4a60-b845-9b5af9f488d3 }, name: Name(\"external-dns-eed3f3eb-69ed-4a60-b845-9b5af9f488d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 75])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7ac18da-6ad1-4f67-865c-3b950e74806e, kind: Service { id: 62e81a8e-9f5e-4f8b-ab77-1b6098b89825 }, name: Name(\"external-dns-62e81a8e-9f5e-4f8b-ab77-1b6098b89825\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 252, 149])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1a5e6457-fc9a-493a-951a-641094dc879e, kind: Service { id: fc1b1f72-cd47-46c1-810c-f9761356930c }, name: Name(\"nexus-fc1b1f72-cd47-46c1-810c-f9761356930c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 147])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5b2a0469-638e-41fd-a7ee-35b54de67ea1, kind: Service { id: d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd }, name: Name(\"nexus-d798d81a-4c8a-4b2f-99a0-a7f78ebf55dd\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 211, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fb30fca2-0efb-4d2d-b64e-315624fdc494, kind: Service { id: 624690b4-cee6-4070-81e8-611b39bdcdb3 }, name: Name(\"nexus-624690b4-cee6-4070-81e8-611b39bdcdb3\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 231, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731846392Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731870502Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731884882Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731901892Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731933033Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731939493Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.731965503Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ea28c4c8-8092-457c-abbd-71ccc879d1a2","remote_addr":"[fd00:1122:3344:101::a]:55655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":360,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"71d8d598-5943-482e-8088-e8e01a5c479b\", \"content-length\": \"127\", \"date\": \"Wed, 13 Mar 2024 04:57:41 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"71d8d598-5943-482e-8088-e8e01a5c479b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.795870717Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.905694785Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.905826598Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"c2484fdd-3feb-43ec-b883-27c72e59efd8","remote_addr":"[fd00:1122:3344:101::a]:33281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":111,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.906072781Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.906131202Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.906484168Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:41.907846419Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"074700e8-895d-4f18-96ba-3613a53c823c","remote_addr":"[fd00:1122:3344:101::a]:33281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1852,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.724463966Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53802"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.731004378Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:505","metadata":"InstanceMetadata { silo_id: 366c886d-6617-4593-8b2c-af862ed2c297, project_id: 144c544d-20a5-4772-af38-01963e782094 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-13T04:57:51.387065Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5114785))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-8f431ebc319\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a27dab12-3d34-4efa-a6d9-cb137a3ce5a0, read_only_parent: Some(Volume { block_size: 512, id: 2b04c09b-5d5f-4853-b17c-2dbacdd99c37, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 20136bd3-7ea9-47a5-9224-428ea35b0be2, key: Some(\"AtjQGcQ20lgK5hVvjaGh9OB/oT7ZlHO4cN4pqSUYbLA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a27dab12-3d34-4efa-a6d9-cb137a3ce5a0, key: Some(\"zJ1cwHIcRezpD7sSuH8hOzgE2OJSMPNilpTy6nv+CaY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19002\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACoOSdtWG1YAAA5J21YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAqDknbVhtWAAAOSdtWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5MWRjMDNmYy01ZGE3LTRlNmYtODBmYS0xOGE2YzlmZWE1M2YiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQTFmOWtsbWRXL1d2bFBDWDFZc3lEanIyQ3dCKzhybExkNWxqWmFDL1dzZCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.73111794Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:539","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.731476395Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5114785))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: d263d960-031f-4403-a332-bb2a4ce72759, kind: Instance { id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 122, 151])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5114785), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-8f431ebc319\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a27dab12-3d34-4efa-a6d9-cb137a3ce5a0, read_only_parent: Some(Volume { block_size: 512, id: 2b04c09b-5d5f-4853-b17c-2dbacdd99c37, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 20136bd3-7ea9-47a5-9224-428ea35b0be2, key: Some(\"AtjQGcQ20lgK5hVvjaGh9OB/oT7ZlHO4cN4pqSUYbLA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a27dab12-3d34-4efa-a6d9-cb137a3ce5a0, key: Some(\"zJ1cwHIcRezpD7sSuH8hOzgE2OJSMPNilpTy6nv+CaY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19002\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACoOSdtWG1YAAA5J21YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAqDknbVhtWAAAOSdtWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5MWRjMDNmYy01ZGE3LTRlNmYtODBmYS0xOGE2YzlmZWE1M2YiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQTFmOWtsbWRXL1d2bFBDWDFZc3lEanIyQ3dCKzhybExkNWxqWmFDL1dzZCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-13T04:57:51.387065Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.732789546Z","hostname":"sock","pid":1671,"uri":"/instances/91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","method":"PUT","req_id":"88c9f913-916e-480c-b371-6e20651e3008","remote_addr":"[fd00:1122:3344:101::a]:53802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8279,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.837540415Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58315"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.861810903Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 241, 122, 151]), slot: 0, vni: Vni { inner: 5114785 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.862105557Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.897870003Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:51.919394998Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:54.105533809Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:54.105899734Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 69.687401ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:54.290872282Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 88.617248ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:54.374180177Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 213.868292ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:54.47597343Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 515.070707ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:54.705741134Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 460.299002ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:55.231124355Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 1.101639006s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:55.701384559Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 1.243445363s","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:56.813290322Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 687.487361ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:58.066941721Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:57:58.783251391Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:1425"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 63.691516ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:58.793792695Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 53.846437ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:58.868087211Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 103.84711ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:58.931200502Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 224.299119ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:59.047041894Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\") failed: Property not found. retry in 793.781052ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:57:59.281219566Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.087865632Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:1434"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.122997468Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:1448"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACoOSdtWG1YAAA5J21YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAqDknbVhtWAAAOSdtWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI5MWRjMDNmYy01ZGE3LTRlNmYtODBmYS0xOGE2YzlmZWE1M2YiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQTFmOWtsbWRXL1d2bFBDWDFZc3lEanIyQ3dCKzhybExkNWxqWmFDL1dzZCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-8f431ebc319\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a27dab12-3d34-4efa-a6d9-cb137a3ce5a0, read_only_parent: Some(Volume { block_size: 512, id: 2b04c09b-5d5f-4853-b17c-2dbacdd99c37, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 20136bd3-7ea9-47a5-9224-428ea35b0be2, key: Some(\"AtjQGcQ20lgK5hVvjaGh9OB/oT7ZlHO4cN4pqSUYbLA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::1a]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a27dab12-3d34-4efa-a6d9-cb137a3ce5a0, key: Some(\"zJ1cwHIcRezpD7sSuH8hOzgE2OJSMPNilpTy6nv+CaY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::16]:19000\", \"[fd00:1122:3344:101::13]:19002\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 91dc03fc-5da7-4e6f-80fa-18a6c9fea53f, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.124177267Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:686"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.269884983Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.270263429Z","hostname":"sock","pid":1671,"uri":"/instances/91dc03fc-5da7-4e6f-80fa-18a6c9fea53f/state","method":"PUT","req_id":"060eecdb-baef-4471-a6eb-390dd4c859d1","remote_addr":"[fd00:1122:3344:101::a]:58315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8432617,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-03-13T04:58:00.270373950Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.270410921Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.270429301Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-13T04:58:00.270373950Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }","propolis_id":"3dd19376-faf8-4a4f-8755-da46205ee77a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.270442591Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }, propolis_id: 3dd19376-faf8-4a4f-8755-da46205ee77a, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-13T04:58:00.270373950Z } }","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-03-13T04:58:00.335391132Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.335415432Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.335471963Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-13T04:58:00.335391132Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }","propolis_id":"3dd19376-faf8-4a4f-8755-da46205ee77a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:00.335486603Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }, propolis_id: 3dd19376-faf8-4a4f-8755-da46205ee77a, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-13T04:58:00.335391132Z } }","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:01.863680101Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"0b75c462-55c9-4d15-85cf-4addb4a4b543","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":605,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:09.477464838Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:09.47760761Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"9402342f-74db-4378-b63b-bd9cd13ea893","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45368","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.380899033Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.381461612Z","hostname":"sock","pid":1671,"uri":"/instances/91dc03fc-5da7-4e6f-80fa-18a6c9fea53f/state","method":"PUT","req_id":"a329c4dd-ce00-48d7-a08b-a5e9e88187d9","remote_addr":"[fd00:1122:3344:101::a]:39347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":512,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-03-13T04:58:25.381574264Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.381605854Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.381643635Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-13T04:58:25.381574264Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }","propolis_id":"3dd19376-faf8-4a4f-8755-da46205ee77a"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.381681165Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(3dd19376-faf8-4a4f-8755-da46205ee77a), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-13T04:57:51.411137623Z }, propolis_id: 3dd19376-faf8-4a4f-8755-da46205ee77a, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-13T04:58:25.381574264Z } }","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-03-13T04:58:25.400793243Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.400811893Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.400827743Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-03-13T04:58:25.400793243Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-13T04:58:25.400793243Z }","propolis_id":"3dd19376-faf8-4a4f-8755-da46205ee77a"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.400838473Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:607","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.400843183Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.401982731Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.402034232Z","hostname":"sock","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:25.981294581Z","hostname":"sock","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1235","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a\", bundle_id: 3921baa5-bab9-4839-886a-768b31f4c5d0 }, time_created: 2024-03-13T04:58:25.402173954Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_3dd19376-faf8-4a4f-8755-da46205ee77a","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:58:25.981378463Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:27.740646225Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:27.767848928Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-13T04:58:25.400793243Z }, propolis_id: 3dd19376-faf8-4a4f-8755-da46205ee77a, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-13T04:58:27.767789227Z } }","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:27.851791183Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46194"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-03-13T04:58:27.852961932Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:27.852981732Z","hostname":"sock","pid":1671,"uri":"/v2p/d263d960-031f-4403-a332-bb2a4ce72759","method":"DELETE","req_id":"40478506-f7ba-4eb2-bf02-9f5370a62556","remote_addr":"[fd00:1122:3344:101::c]:46194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1112,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:27.989984543Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-13T04:58:25.400793243Z }, propolis_id: 3dd19376-faf8-4a4f-8755-da46205ee77a, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-13T04:58:27.767789227Z } }","instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:27.99434205Z","hostname":"sock","pid":1671,"instance_id":"91dc03fc-5da7-4e6f-80fa-18a6c9fea53f","component":"InstanceManager","file":"sled-agent/src/instance.rs:724"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:31.862975029Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/d2af1a47-eaf4-438c-b9b1-fd4ec305f6c9","method":"GET","req_id":"eee1449c-9b2a-485b-a73f-d0eefec0bb81","remote_addr":"[fd00:1122:3344:101::d]:62491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":520,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:32.061985654Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:32.062075505Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-13T04:58:32.099149082Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.99999601s","next_cleanup":"Instant { tv_sec: 2565, tv_nsec: 382391332 }"}