[ Feb 9 21:11:12 Enabled. ] [ Feb 9 21:11:12 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Feb 9 21:11:12 Method "start" exited with status 0. ] [ Feb 9 21:11:12 Rereading configuration. ] [ Feb 9 21:11:13 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.004801447Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:247"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.060129462Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:254","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.060165313Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.172435304Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.18149469Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.189267173Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:264","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.79327004Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.931300344Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.931339645Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.931467277Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.93159295Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.93163498Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.931650371Z","hostname":"sock","pid":1671,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.931716532Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934662616Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:13.934668866Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934685496Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934694416Z","hostname":"sock","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934732197Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934744468Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934751228Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934756118Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934760608Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934764618Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934770488Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934775548Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934779338Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934785498Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934790198Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"None"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934798689Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934802538Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:13.934912871Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.130606962Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.130654623Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.130668933Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304629897Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304686848Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304705898Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304713678Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304722188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304728739Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304735599Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304754049Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304763399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304774669Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.30478116Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.30480067Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.30481687Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 494592, free: 20937470976, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 548352, free: 20937417216, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304873321Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304962073Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304976843Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.304989193Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.305001744Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.305023374Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.305030224Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.305072125Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.305080235Z","hostname":"sock","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.319699523Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:697"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.319729694Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999870967s","next_cleanup":"Instant { tv_sec: 3016, tv_nsec: 147536435 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.319782605Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3345"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.319854966Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.319913137Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.319928198Z","hostname":"sock","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.32331182Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323412551Z","hostname":"sock","pid":1671,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323670846Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323679796Z","hostname":"sock","pid":1671,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323739997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323794198Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323818839Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323823539Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323840389Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.32385663Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.32386712Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.32388074Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.32388463Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.32388838Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323917751Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.323935011Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.324700035Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.324872858Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.324885918Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.324937859Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:556"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.324947Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325127503Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325262185Z","hostname":"sock","pid":1671,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:640"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325318437Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e8553772-9db2-44b1-acc1-a08a65e40999, rack_id: 56f2c6d3-9416-4aae-9f83-77cafa4b4cdb, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325429408Z","hostname":"sock","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:41545","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: e8553772-9db2-44b1-acc1-a08a65e40999, rack_id: 56f2c6d3-9416-4aae-9f83-77cafa4b4cdb, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.32553229Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325544201Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325557941Z","hostname":"sock","pid":1671,"file":"sled-agent/src/server.rs:47"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325572771Z","hostname":"sock","pid":1671,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325597212Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.325604522Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.327946145Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3573"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.333871963Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.333940495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.333946385Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.333977485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.334059197Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.334063967Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.364538877Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.372287309Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.4067068Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:339"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.406747211Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.474863922Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.496858255Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.496913776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.496962357Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.497002818Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.497132411Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.497148261Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.525043353Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.532610912Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.636415697Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.658261088Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.658313559Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.658323239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.65833517Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.658482402Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.658489933Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.684798315Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.692093539Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.793331118Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.815129098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.815174379Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.815180929Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.81522145Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.815375122Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.815385673Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.842798936Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.85012454Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.949756989Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.971743952Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.971797283Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.971804103Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.971833834Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.971973646Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:14.971980836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.000618372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.008332654Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.116798365Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.138746548Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.138802589Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.138827399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.13886987Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.138987002Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.138993992Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.164356758Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.17155915Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.274380118Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.295952014Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.295996814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.296006195Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.296015775Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.296156937Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.296164157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.324314384Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.331580148Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.43687783Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.458619289Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.45866498Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.458697581Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.458708911Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.458845294Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.458857364Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.485670526Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.492886498Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.595199247Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.616246033Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.616291724Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.616301124Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.616319624Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.616455577Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.616471627Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.645206755Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.652764043Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.758856921Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.77354143Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.773597082Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.773612402Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:351"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.778244717Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 1218048, free: 20936747520, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 494592, free: 20937470976, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 548352, free: 20937417216, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 775680, free: 20937189888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 778752, free: 20937186816, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 781824, free: 20937183744, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.796460251Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.799089149Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.813678727Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.821640693Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.821680894Z","hostname":"sock","pid":1671,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.8263782Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.826389251Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.826394561Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:15.887562954Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.106825547Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.119800196Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.119890407Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.119922438Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.119938568Z","hostname":"sock","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.28566115Z","hostname":"sock","pid":1671,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.821196271Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.821626539Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.82166008Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:178"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.82167937Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:186"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.839029288Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.839237952Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.839548608Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.839937195Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.840092208Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.840346243Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.840460835Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.840564727Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.840694299Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.840823471Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.840935003Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.841061506Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.841188418Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.84131594Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.841434743Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.841562295Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.841678387Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.841786789Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.841911551Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.842028754Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.842138725Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:16.842251258Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:19.704024508Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 40.877709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:19.940637112Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 142.171631ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:20.135176553Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 233.272744ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:20.309272409Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 508.032236ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:20.675116315Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 975.799774ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:21.247884228Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.102738312s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:22.426085435Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.230008503s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:23.570009833Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:26.050511955Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1847"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:26.947773266Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:978","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:26.947847707Z","hostname":"sock","pid":1671,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006086495Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006125116Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006131856Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006172797Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006201297Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:27.006300609Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"143.694891ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006502833Z","hostname":"sock","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006694016Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:137"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006721487Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006745787Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3345"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.006752378Z","hostname":"sock","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:41545","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.009553389Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.0096161Z","hostname":"sock","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.009810724Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.009903955Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.009960516Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.010023827Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.010135009Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:704","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.042088536Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:62358"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.042374991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.042461973Z","hostname":"sock","pid":1671,"uri":"/zpools","method":"GET","req_id":"04a08602-b14d-435f-8339-399a4b954fe6","remote_addr":"[fd00:1122:3344:101::1]:62358","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":221,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.071128759Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.071956114Z","hostname":"sock","pid":1671,"uri":"/sled-role","method":"GET","req_id":"d928b417-b3e0-44f5-af3e-5be6145913a4","remote_addr":"[fd00:1122:3344:101::1]:47883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":797,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.073857079Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.07386717Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.074160335Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.074359208Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:748"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.077377484Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.109561175Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.109920301Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33613"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.110256278Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.110271368Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:27.153726705Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"315.502872ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.163186649Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.16320509Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.18721399Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.214889978Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.217120299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.263261256Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.263316937Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.263333297Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.263391969Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.281479401Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.308206621Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.308240022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.308246052Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.360190526Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.364153208Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.368379016Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.380908276Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.385167584Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.389098626Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:27.47045627Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"1.290584251s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.503318423Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.618058849Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:27.910781462Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:28.124556246Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:28.360824953Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:28.762202201Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"2.699949697s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:29.000570886Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:29.069718266Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:29.184019904Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 40.117727ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.205700052Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:29.210246725Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 72.999527ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.253612041Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 163.225053ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.346348944Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 25.203917ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.409061875Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 112.450329ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.517900233Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:29.518579575Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 520.815174ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.58205085Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 37.177322ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.603563095Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 289.325007ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.651399153Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 134.232882ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.697626992Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 245.853691ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.847256109Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:29.908536064Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 548.470435ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:29.951701876Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 570.648744ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:30.102022084Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 535.774669ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:30.110798145Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:30.290365252Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 725.727289ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:30.508820452Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 1.299894732s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:30.65689833Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 597.057589ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:30.680693657Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:31.10307437Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1864"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:31.10308894Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 771.246225ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:31.243791683Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 923.59601ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:31.286206501Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:31.365352414Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:31.46372916Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"2.875091149s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 766.98696ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:31.967832283Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 1.323907058s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:32.024239549Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:32.081355507Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 1.468136606s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:32.219804859Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 1.450467077s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:32.744514701Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:32.771606148Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 607.085301ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:33.356098586Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:33.565309127Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 1.343537151s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:33.696336152Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 744.573ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:33.970887711Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac\") failed: Property not found. retry in 1.488105957s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:34.2028684Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:34.340353214Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"6.370950409s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 534.382286ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:34.724450954Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df\") failed: Property not found. retry in 1.114424037s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:35.049326747Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff\") failed: Property not found. retry in 1.212186511s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:35.267873469Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe5b:80f9","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:35.635544958Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1871"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:35.642849572Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:35.732516628Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:35.732553529Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:35.803142085Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2404"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.107810446Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2283"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.285479908Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.285517369Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.43541351Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2318"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.488562616Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.637116643Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2585"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.782459431Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2590"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.869771364Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:36.965899338Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2524"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.132316383Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2312"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.14200186Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3597","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.142075891Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3613"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.142091102Z","hostname":"sock","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.444009694Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.465760003Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:2::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.465803364Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.687584745Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3621"} {"msg":"configuring dendrite zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.693804019Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3677"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.874478966Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3717","rack_subnet":"fd00:1122:3344::"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:37.925472842Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 68.868848ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:37.957352057Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ce9ef48b-b5e5-49d6-b270-5df5305e095f\", \"content-length\": \"152\", \"date\": \"Fri, 09 Feb 2024 21:11:37 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"ce9ef48b-b5e5-49d6-b270-5df5305e095f\" }"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.010114135Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.010173046Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Failed to get switch ID from MGS (retrying in 133.636646ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:38.027627707Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70d70311-18d0-40f1-ab3f-937a0241f0f2\", \"content-length\": \"152\", \"date\": \"Fri, 09 Feb 2024 21:11:37 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"70d70311-18d0-40f1-ab3f-937a0241f0f2\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 280.278733ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:38.162840259Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4430e0bb-aa7e-4dba-b099-ddb582df4d23\", \"content-length\": \"152\", \"date\": \"Fri, 09 Feb 2024 21:11:37 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"4430e0bb-aa7e-4dba-b099-ddb582df4d23\" }"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.421142221Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.443809927Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.475810414Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.475889906Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.476278453Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.476319093Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.554316685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.996532862Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:3::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:38.996573133Z","hostname":"sock","pid":1671,"zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:39.422089454Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:39.543523353Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:39.543581264Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:39.95029615Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.429901863Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.503119737Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_0e86c47c-f102-4dae-b3de-66cf774453ac"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.503166128Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_10efe7b5-4c14-431b-ae96-fd6d968a40df"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.503175938Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_56b50ae5-31a2-4588-b795-6af274c2aaff"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.503216039Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.503326031Z","hostname":"sock","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.503550845Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.50383111Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"e791d740-249b-40bb-9afb-6e2943e086d9","remote_addr":"[fd00:1122:3344:101::1]:33613","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":13393824,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.505523741Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.56103472Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.621177894Z","hostname":"sock","pid":1671,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.683807484Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.683826774Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.715008357Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.716984253Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.716990063Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.745372014Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.745566508Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64700"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.746459024Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.746471174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.767739955Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.767822066Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.788667529Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.78872046Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.808104596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.808169727Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.808379611Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.808503453Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.820605935Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.852969319Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.854738632Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.854749442Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.899940782Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", _ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 174, 184]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.899969222Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.900480472Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.900487641Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.900760806Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.902751063Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.927434516Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:40.945404186Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:41.726411662Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34370"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:41.726530694Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"91803a58-c284-43c2-907e-fe561071e6e7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34370","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":52,"response_code":"200"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:41.968912843Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 50.137181ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:42.1092873Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 67.763527ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:42.167225863Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 169.531492ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:42.251075823Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:42.251153754Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 282.804716ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:42.432973182Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 1.11098425s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:42.726446959Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 1.22273492s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:43.848706549Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 791.208201ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:45.081930446Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 957.785455ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:45.881207848Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 663.346834ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:46.848393521Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14\") failed: Property not found. retry in 1.187272299s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:47.521780412Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:48.748045922Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::1b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:48.748103572Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:49.29388655Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:49.293935111Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Set up NTP service boundary=true, Servers=[\"0.pool.ntp.org\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:50.001753584Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2209"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:50.050974267Z","hostname":"sock","pid":1671,"zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.478049317Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.478149059Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.478442824Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.478661578Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1ab3679c-7373-4034-8221-c7148795ebc9","remote_addr":"[fd00:1122:3344:101::1]:64700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":14733030,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.481053272Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.50979016Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.509938642Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43575"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_71d25d50-2cba-4615-b456-83b65add8d14': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.519670841Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.519791693Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"b9326331-36de-4c7e-a02f-dc035ca055c2","remote_addr":"[fd00:1122:3344:101::1]:43575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9718,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:55.522560164Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.614978611Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.615082273Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.624172619Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"6e2ebdb3-8a76-4137-91ad-43f08206226c","remote_addr":"[fd00:1122:3344:101::1]:61465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8992,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.627127834Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:55.627133864Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.726693411Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.726815793Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.73532048Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"d3781bdb-7b64-4f64-af9f-cad7c535c5d6","remote_addr":"[fd00:1122:3344:101::1]:53231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8308,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.737088992Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:55.737100842Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.978440852Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.978563214Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.986916017Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"24bb3f87-9018-4fba-bb39-6952729926a4","remote_addr":"[fd00:1122:3344:101::1]:50109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8193,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:55.988816652Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:55.988822732Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.200671461Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.200846565Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54007"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.210189106Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"d10543c2-50e2-4f4b-a783-713e54ed4606","remote_addr":"[fd00:1122:3344:101::1]:54007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9225,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.212109731Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:56.212114101Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.465437002Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.465572244Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51590"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.474231813Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"8640dc11-0031-4b7b-a7da-39c1295b5a57","remote_addr":"[fd00:1122:3344:101::1]:51590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8532,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.476165678Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:56.476171988Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.678719357Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.67889418Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57441"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.687504968Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"9f7443c2-5193-41db-8475-a62267253603","remote_addr":"[fd00:1122:3344:101::1]:57441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8555,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:56.69087384Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:56.69088157Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.04285338Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.043016933Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64292"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.052863544Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"40622eb9-353c-46ab-8464-67a3be7e02c2","remote_addr":"[fd00:1122:3344:101::1]:64292","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9774,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.054522414Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:57.054528144Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.446839496Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.44703227Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:36366"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.455872712Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"b0705f90-9d26-4e78-adcb-e932e62ad106","remote_addr":"[fd00:1122:3344:101::1]:36366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8785,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:57.459631861Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:57.459642181Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.112936143Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.113046405Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54993"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.121482189Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"b5f9ac5f-bec8-4cd4-b516-45c2c1cde988","remote_addr":"[fd00:1122:3344:101::1]:54993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8307,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.123554887Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:58.123560138Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.688275704Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.688399356Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.697554284Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"57e7de41-cf45-45ae-9cdc-c170ce3f066b","remote_addr":"[fd00:1122:3344:101::1]:34710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8985,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:58.699228845Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:58.699234545Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:59.419177Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:59.419283862Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:36194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:59.427656966Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"4faa40cc-c167-447e-bbf3-85252fa2b630","remote_addr":"[fd00:1122:3344:101::1]:36194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8239,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:11:59.429681473Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:11:59.429688873Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:00.268287707Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:00.268404689Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42471"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:00.277627328Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"1249cbd7-7119-4e5e-94f8-b7f1c9476378","remote_addr":"[fd00:1122:3344:101::1]:42471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9086,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707513114.070866, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:00.279742917Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:00.279747877Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.277580836Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.277687318Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55935"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.28106471Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.298955038Z","hostname":"sock","pid":1671,"uri":"/timesync","method":"GET","req_id":"947b5541-7124-4698-984b-75fec83fc5e4","remote_addr":"[fd00:1122:3344:101::1]:55935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":21056,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2616771612, ip_addr: 155.248.196.28, stratum: 3, ref_time: 1707513120.4439783, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.302822159Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.302827559Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.332395261Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.332619745Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57345"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.33287311Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.33288813Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.352912978Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.352965169Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.371843545Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.371903007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.390194272Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.390244813Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.46796518Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.468013301Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.57636883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.576429671Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.685437972Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.685491393Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.784360438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.784413619Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.871415696Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.871470847Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.871680851Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.871809363Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:01.905055023Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:01.905100584Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:01.905108644Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:01.905115195Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:01.905123135Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:01.905281678Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d2c63407-128e-4fbb-aca1-f76cdd0089a6","remote_addr":"[fd00:1122:3344:101::1]:57345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\", TimeNotSynchronized), (\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\", TimeNotSynchronized), (\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\", TimeNotSynchronized), (\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\", TimeNotSynchronized), (\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\", TimeNotSynchronized)]","latency_us":572602,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:01.905598483Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"133.113098ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2c63407-128e-4fbb-aca1-f76cdd0089a6\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2c63407-128e-4fbb-aca1-f76cdd0089a6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.039707055Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.041247863Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.041263904Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.060094229Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.060173711Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.07975769Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.079806111Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.098308451Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.098374582Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.117619695Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.117688737Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.135391572Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.135451723Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.154599164Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.154667525Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.174163223Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.174232355Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.191857508Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.191929989Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.192193544Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.192398668Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.225889473Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.225934944Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.225941764Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.225949604Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.225956094Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.226101407Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"6fc0cd3d-9e5e-48cf-8e35-694a1243e50e","remote_addr":"[fd00:1122:3344:101::1]:57345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\", TimeNotSynchronized), (\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\", TimeNotSynchronized), (\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\", TimeNotSynchronized), (\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\", TimeNotSynchronized), (\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\", TimeNotSynchronized)]","latency_us":185868,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.226311651Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"160.449863ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6fc0cd3d-9e5e-48cf-8e35-694a1243e50e\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6fc0cd3d-9e5e-48cf-8e35-694a1243e50e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.387530179Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.388197662Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.388223832Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.410684224Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.410723305Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.430276734Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.430332835Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.448432907Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.448482378Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.467916345Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.467976646Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.487598366Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.487647937Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.506256779Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.50629535Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.525610724Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.525663165Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.54393679Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.543978731Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.544171155Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.544294857Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.578508565Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.578564486Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.578574686Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.578582426Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.578592157Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.578721359Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8e97cfc2-8ded-476a-bc3f-3df02020fdf0","remote_addr":"[fd00:1122:3344:101::1]:57345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\", TimeNotSynchronized), (\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\", TimeNotSynchronized), (\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\", TimeNotSynchronized), (\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\", TimeNotSynchronized), (\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\", TimeNotSynchronized)]","latency_us":190721,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:02.578953393Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"103.684713ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e97cfc2-8ded-476a-bc3f-3df02020fdf0\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e97cfc2-8ded-476a-bc3f-3df02020fdf0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.684082713Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.684759395Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.684795876Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.704377195Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.704434767Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.721328527Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.721376088Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.739797966Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.739846107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.756506092Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.756569714Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:02.775252427Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.000315146Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.002094049Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.018681653Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.018735854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.036544391Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.036610312Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.055345186Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.055400137Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.055660462Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.055776044Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.138775487Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.138863119Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.138879359Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.13889286Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.13890498Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.234330152Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.238649471Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.242843528Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.247171207Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.251981905Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.256928836Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.261277716Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.265526114Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.270320522Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:03.275624299Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:06.539081423Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:06.539218885Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:06.552713223Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:06.552852626Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:06.591948084Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:06.592109107Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 28.083965ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:06.692480849Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 64.50543ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:06.700069058Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 123.559251ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:06.729728853Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 68.380483ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:06.736945695Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 141.354729ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:06.891065264Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 177.673338ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:07.433871308Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 209.931831ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:07.595683988Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 103.171047ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:07.713412319Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 390.978319ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:07.822101314Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 329.106421ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:07.845517494Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 285.657618ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:07.867542748Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:07.879509308Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:07.880279072Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 397.931533ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.165683821Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 1.138492914s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.187572413Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 69.653656ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.207659202Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.140149871s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.224040692Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 133.613321ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.337964894Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:08.338600386Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:08.338783809Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 255.844678ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.864333096Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 664.460474ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.899855858Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 291.362147ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:08.982171009Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 68.474373ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.092499454Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 102.8834ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.181422706Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 601.681469ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.300057394Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 296.436058ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.312425191Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 1.135237736s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.346800182Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 877.384427ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.389242591Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.296974006s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.57876833Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 563.906273ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.626443115Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 1.171114424s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:09.912342173Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 523.925816ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:10.20090094Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.304261457s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:10.277362354Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 845.911253ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:10.492110025Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 1.022598715s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:10.734484104Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 733.586464ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:10.885341003Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 913.453573ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:11.094114636Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 695.74415ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:11.347501307Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 715.454569ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:11.590825833Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.096130312s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:11.628557776Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 796.07768ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:11.764685634Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 1.051700401s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:12.017607687Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 1.014822151s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:12.052393156Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 993.241964ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:12.315277062Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:12.323876949Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:12.324034632Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"44d2d410-1d41-4168-aa5f-694e548f6e56","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62266","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":81,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 767.155282ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:12.569677671Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 631.981894ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:12.733775923Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 517.320346ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.079482089Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 772.939935ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.090957309Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.113369243s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.322474269Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 513.249935ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.350545675Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 576.055508ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.378267574Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 886.233229ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.608368477Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 891.924889ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.87601631Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 1.457824353s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.886002583Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.450259942s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:13.96592554Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.215909986s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:14.445371961Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 640.567003ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:14.503259014Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 1.106023869s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:14.776085382Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 1.480453815s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:15.152843418Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 666.747033ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:15.352546474Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.31391361s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:15.423821612Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.016100327s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:15.671975186Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3\") failed: Property not found. retry in 1.398474154s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:15.891934744Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 685.328915ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:16.029287275Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728\") failed: Property not found. retry in 1.156089235s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:16.643790915Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.420983789s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:16.697682924Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c\") failed: Property not found. retry in 811.122194ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:16.724731271Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.367333765s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:16.74754014Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 814.806627ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:18.125639216Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.352006637s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:18.134719783Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 816.797855ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:18.951410254Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 965.949321ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:19.497158942Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 968.114505ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:20.3828692Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 835.825977ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:20.484430794Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.283146515s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:21.332201346Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.392981016s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:21.363021322Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 697.824988ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:22.749611084Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 789.976026ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:22.773247208Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.030682683s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:23.457106331Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.046463059s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:23.574326572Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.2785774s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:24.497349896Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.135513726s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:24.630126552Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 1.129339581s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:25.774625571Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 971.924175ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:25.784013003Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f\") failed: Property not found. retry in 1.012381081s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:26.765920107Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777\") failed: Property not found. retry in 561.763852ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:26.913617568Z","hostname":"sock","pid":1671,"zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.801126799Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_ef72eb88-de65-4f60-8270-28e9b6c1431f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.801217291Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_9822dded-9084-4bcb-acff-e08ee8809728"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.801230901Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_93bdd1d6-923f-4556-a2e7-c8932e375777"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.801239251Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_50e50144-887b-4bfb-9c64-0dab98e559b3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.801249131Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_0bcc78b8-eeff-44dc-a9db-f18fef587e2c"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.801298412Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.801847113Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.80278591Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a9d2b91c-ef3a-407d-936e-ca0860e91717","remote_addr":"[fd00:1122:3344:101::1]:57345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":24893260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.836735083Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51527"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:27.86380329Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3165"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:28.054875977Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3183"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.206216889Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3206"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.206324421Z","hostname":"sock","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"172b344a-03dc-4068-b946-4bd91fdf8c5f","remote_addr":"[fd00:1122:3344:101::1]:51527","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":4369546,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.240117291Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.240534049Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49975"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.240919196Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.240926686Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.262285209Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.26234523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.280046775Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.280108336Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.299843538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.299903609Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.317088044Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.317146465Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.335127246Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.335179677Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.353406901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.353463862Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.371113316Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.371173747Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.388780751Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.388839372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.458986409Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.4590457Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.519455299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.51951187Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.575538058Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.575599959Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.634335207Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.637157489Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.690654651Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.690710982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.760156967Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.760212488Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.815481523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.815532733Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.870145616Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.870208597Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.926936709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.928985806Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.987032292Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:32.988999538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.04848228Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.048542761Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.104149552Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.104213783Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.104429777Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.104564269Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202161591Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202212152Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202220002Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202226212Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202232652Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202238542Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202244352Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202248622Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202254083Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202265733Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202272103Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202276353Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202280923Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202286883Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202293353Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202297703Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202304484Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202310073Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202315484Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.202468147Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c9fc1864-8ed5-4f98-b981-a51bf9d6c94e","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized)]","latency_us":961818,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.202716391Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"86.117775ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c9fc1864-8ed5-4f98-b981-a51bf9d6c94e\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c9fc1864-8ed5-4f98-b981-a51bf9d6c94e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.289886711Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.290579264Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.290596844Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.309492031Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.309537802Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.32793055Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.32797495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.346134194Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.346175434Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.364649044Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.364707335Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.382688445Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.382732056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.400605024Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.400652474Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.418690316Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.418733596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.43638528Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.436428681Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.455180406Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.455226456Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.473265397Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.473307078Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.491754747Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.491809958Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.510938589Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.51098254Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.529125382Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.529165413Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.547030731Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.547070552Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.564532792Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.564580373Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.582982641Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.583028721Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.600416741Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.600455412Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.618467982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.618506993Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.636411392Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.636454872Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.654451853Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.654509874Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.654713848Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.65486982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749419006Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749457697Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749463117Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749467507Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749471757Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749477537Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749483727Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749497187Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749502768Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749508557Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749515598Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749522978Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749526918Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749533808Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749539308Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749544868Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749550248Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749557618Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749563209Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.749692551Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"0fd781d2-4c67-49ce-9655-b00c1b41ba3a","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized)]","latency_us":459279,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:33.749926515Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"157.392ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0fd781d2-4c67-49ce-9655-b00c1b41ba3a\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0fd781d2-4c67-49ce-9655-b00c1b41ba3a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.909102877Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.909740879Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.909754169Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.931358776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.931429377Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.951818831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.951870242Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.972275597Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.972329178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.99042801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:33.990481311Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.00783349Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.007889331Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.025392292Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.025445193Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.045722225Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.045776966Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.064318807Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.064382888Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.082845556Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.082901788Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.102938716Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.102988986Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.120925936Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.120980127Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.138527259Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.13857773Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.157462146Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.157511387Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.174885816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.174937667Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.192673933Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.192732174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.210680643Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.210731274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.230460166Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.230519177Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.24919995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.249267631Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.266914466Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.266966967Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.285173111Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.285257892Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.285474406Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.285610159Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377605618Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377655988Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377668308Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377673379Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377690389Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377695129Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377700539Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377706349Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377710649Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377714289Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.37771866Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.377722369Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.37772583Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.37772931Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.37773292Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.3777363Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.37774044Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.37774491Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.37774919Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.377816791Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"22cc2aea-2bce-4f08-bf0d-c0a50bb555f3","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized)]","latency_us":468280,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.378041626Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"87.288153ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"22cc2aea-2bce-4f08-bf0d-c0a50bb555f3\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"22cc2aea-2bce-4f08-bf0d-c0a50bb555f3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.466411668Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.467389895Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.467402696Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.486770671Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.486812912Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.50466031Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.504700791Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.523243371Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.523282572Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.541592717Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.541642778Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.560035236Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.560082836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.577072538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.577115409Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.594273694Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.594320515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.612258154Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.612295245Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.63053185Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.630571661Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.646798768Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.646845289Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.66489798Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.664936591Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.681164619Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.68120554Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.699229061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.699268971Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.71663715Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.716682441Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.735090479Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.73512823Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.751406689Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.751440509Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.768571844Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.768605654Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.78634828Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.786389541Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.804191397Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.804233658Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.822155877Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.822204938Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.822422152Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.822579065Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914073055Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914139936Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914171107Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914177596Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914184087Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914190557Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914198027Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914203237Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914210467Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914216797Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914223057Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914229048Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914234888Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914242368Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914248328Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914255268Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914261358Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914287998Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914301059Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:34.914419161Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"396971b2-55b6-499a-a3fa-799369f18b28","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized)]","latency_us":447615,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:34.914674266Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"249.440784ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"396971b2-55b6-499a-a3fa-799369f18b28\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"396971b2-55b6-499a-a3fa-799369f18b28\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.165117833Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.165744654Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.165764775Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.189362758Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.189430629Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.210492006Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.212950011Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.22975336Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.229812781Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.249485252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.249541783Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.269755344Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.269808055Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.289658239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.28972029Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.308725129Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.30878121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.328103495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.328157416Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.347263597Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.347335158Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.366222685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.366277916Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.386008718Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.386065259Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.404954316Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.405015057Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.424084327Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.424145728Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.443167677Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.443225178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.463929258Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.463988649Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.483299984Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.483358275Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.502773121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.502841353Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.523087174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.523153605Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.542377287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.542431348Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.563635688Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.563707929Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.563921123Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.564089076Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658828515Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658905076Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658915097Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658922237Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658936117Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658946007Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658953777Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658959907Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658964707Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658970558Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658984838Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658991368Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.658998868Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.659005288Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.659011738Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.659038989Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.659044969Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.659051139Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.659055749Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.659165961Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7d1ad351-0882-4e12-8cfd-a548357bb6a2","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized)]","latency_us":493608,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:35.659410446Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"144.542263ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7d1ad351-0882-4e12-8cfd-a548357bb6a2\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7d1ad351-0882-4e12-8cfd-a548357bb6a2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.804677372Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.805302514Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.805319064Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.82798617Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.828045901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.846813806Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.846880647Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.864887458Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.867444565Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.883070001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.883132122Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.901115652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.903480496Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.919392198Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.919456429Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.937542561Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.937598842Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.955499201Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.955553152Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.974302776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.974371267Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.99358706Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:35.993647371Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.012142381Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.012198132Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.030405206Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.030465707Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.04859881Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.048663121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.06821233Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.068271421Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.086650729Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.086705679Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.104815202Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.104872233Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.124211328Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.124274049Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.144340628Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.144397618Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.16302068Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.163084452Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.181431388Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.18151872Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.181706193Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.181845886Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283305748Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283370229Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.28337776Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.28338867Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.28339337Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.28339888Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.28340432Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.28342251Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283427591Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283432321Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283436741Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283440511Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283462381Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283471242Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283476831Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283481351Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283485142Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283489542Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283494552Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.283504552Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"40178461-f845-453b-99b0-21db1a41fbec","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized)]","latency_us":478415,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.283739136Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"157.093005ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"40178461-f845-453b-99b0-21db1a41fbec\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"40178461-f845-453b-99b0-21db1a41fbec\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.44193095Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.442549371Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.442563072Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.463873053Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.463940054Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.481728501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.481772331Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.500149919Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.50019854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.51765858Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.517700941Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.539609712Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.539656093Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.559687191Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.559732022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.579694858Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.579741839Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.598141057Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.598185168Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.619636572Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.619681052Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.639128039Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.63918016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.660696785Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.660742716Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.680443998Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.680495249Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.699224082Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.699265373Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.717321995Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.717362466Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.735430367Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.735471418Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.753393887Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.753436778Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.771392667Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.771440948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.79061584Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.790659001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.808432147Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.808473478Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.826687892Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.826738063Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.826955347Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.82708284Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930429717Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930518888Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930529168Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930537359Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930544299Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930552479Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930565419Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.93058008Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.93058694Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.93059308Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.93061713Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.93062468Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930632161Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.93064067Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930648301Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930653981Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930661731Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930667451Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.930674991Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:36.930796564Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"0d6ecadb-250b-45da-975b-030c937c7e35","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized)]","latency_us":488442,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:36.931081299Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"378.212684ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0d6ecadb-250b-45da-975b-030c937c7e35\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0d6ecadb-250b-45da-975b-030c937c7e35\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.311131225Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.311751997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.311763497Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.335430401Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.335481572Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.356554009Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.35660109Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.376909223Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.376957343Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.397928939Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.397984229Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.418497526Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.418539287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.439361379Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.43940471Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.459811924Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.459858775Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.481204597Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.481252178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.501790115Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.501838196Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.522895982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.522946883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.544498238Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.54455325Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.565446073Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.565493634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.587367345Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.587420696Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.609034613Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.609096724Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.630034359Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.63009741Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.650632047Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.650677478Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.670908199Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.67095657Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.69166427Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.691715911Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.712296259Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.71235158Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.732748934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.732799175Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.732993959Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.733117821Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834185276Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834260848Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834268968Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834275068Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834282638Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834289148Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834295388Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834323039Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834329369Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834336459Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834342499Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834354929Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.83436303Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.83436773Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.83437335Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.83438237Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.83438673Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.83439135Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.83439703Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:37.834455611Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"9cb7cc2f-f624-4203-bb96-b310146bd3aa","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized)]","latency_us":522902,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:37.834651255Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"430.81234ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9cb7cc2f-f624-4203-bb96-b310146bd3aa\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9cb7cc2f-f624-4203-bb96-b310146bd3aa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.267387478Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.26801061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.26804099Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.29035401Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.290416801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.311550419Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.31161583Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.331705039Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.33177206Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.352012162Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.352074783Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.371970138Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.374420973Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.391933485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.391999066Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.411416592Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.411498544Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.432717443Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.432776854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.451701782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.451759543Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.472454853Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.472518534Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.493151253Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.493219074Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.513118209Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.51317756Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.535376017Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.535441478Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.55679698Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.556857311Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.576279428Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.576335359Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.598649599Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.598706169Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.618541134Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.618598515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.63958452Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.639659271Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.660916922Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.663343026Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.681319046Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.681413958Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.681606351Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.681753974Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780425545Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780477996Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780484926Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780489226Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780494546Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780524337Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780528377Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780531987Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780535437Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780539797Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780544258Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780547738Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780556718Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780560458Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780564158Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780567648Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780571898Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780580358Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780584118Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:38.780609149Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"67ae0479-c1dc-4fd6-9dd9-f1eda455c261","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized)]","latency_us":512803,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:38.780786632Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"267.235606ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"67ae0479-c1dc-4fd6-9dd9-f1eda455c261\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"67ae0479-c1dc-4fd6-9dd9-f1eda455c261\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.04922151Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.049844041Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.049856121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.074721688Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.074772159Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.09283324Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.092890421Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.112638274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.112695575Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.134070027Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.134114538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.154207337Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.154256658Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.175762422Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.175806723Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.193888755Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.193931386Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.215332249Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.21538245Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.233666165Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.233710296Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.253456649Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.256057126Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.275182378Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.275225458Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.296104462Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.296160123Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.31511116Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.315158321Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.335074547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.335130388Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.355025853Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.355072274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.376281223Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.376327354Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.3957461Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.395794132Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.416510852Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.416554373Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.437149011Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.437192521Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.457455753Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.457506804Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.457700638Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.45782539Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.55592112Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556013252Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556023442Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556058163Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556064913Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556075993Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556089833Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556096233Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556103803Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556109974Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556117514Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556122004Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556127654Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556132274Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556142504Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556146924Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556154454Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556158935Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556171635Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.556220786Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"ba1fb170-621c-4d9e-bde3-136df1ea7265","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized)]","latency_us":506573,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:39.556425309Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"323.695757ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba1fb170-621c-4d9e-bde3-136df1ea7265\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ba1fb170-621c-4d9e-bde3-136df1ea7265\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.880694002Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.881726601Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.881741421Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.903192385Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.903248126Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.923548299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.92360515Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.942216871Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.942266782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.960171331Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.960217152Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.98026805Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.980326561Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.998209399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:39.9982755Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.017545614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.017603815Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.037105833Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.037169394Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.057407926Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.057465507Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.078410191Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.078467652Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.098861507Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.098922688Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.117805405Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.117856926Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.136562949Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.13661514Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.155792192Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.155855523Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.174161729Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.17421666Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.193752129Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.193797909Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.21233993Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.212398431Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.230782188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.230829579Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.249473471Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.249522362Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.267491292Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.267555194Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.267773747Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.26790497Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362345854Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362414795Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362422735Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362428275Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362435535Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362443885Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362450435Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362458076Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362470416Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362476726Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362484126Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362490396Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362496356Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362500756Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362507477Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362512117Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362522667Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362528707Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.362534217Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.362607568Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"eaa3f0fd-5b4e-48d1-8811-e3780b9dbd79","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized)]","latency_us":481484,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:40.363274731Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"560.460234ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eaa3f0fd-5b4e-48d1-8811-e3780b9dbd79\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eaa3f0fd-5b4e-48d1-8811-e3780b9dbd79\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.924622994Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.925272126Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.925300407Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.948074485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.948126786Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.971172039Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.97122627Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.990937521Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:40.990996942Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.009628984Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.009687766Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.030474187Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.030533858Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.049472056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.049528687Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.071073472Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.071134053Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.090730483Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.090798274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.109866035Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.109929526Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.133282154Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.133345066Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.15208123Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.15213422Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.170352665Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.170405976Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.188527668Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.190996764Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.206734813Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.206793794Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.226151839Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.22620999Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.244958084Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.245017336Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.26324098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.263297631Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.281949474Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.282011485Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.302048603Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.302111914Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.320108764Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.320189485Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.3204276Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.320586803Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414601489Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.41466433Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.41468456Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.41469298Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.41469846Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414703591Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414710391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414715551Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414720681Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414726601Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414733081Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414739601Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414757401Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414770582Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414778842Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414783992Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414791212Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414800532Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.414806102Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:41.414814822Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"fbd1a41e-9e89-485a-9978-e3e133280649","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized)]","latency_us":489786,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:41.415021476Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"614.698299ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fbd1a41e-9e89-485a-9978-e3e133280649\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fbd1a41e-9e89-485a-9978-e3e133280649\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.030546335Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.031557743Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.031573403Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.052791883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.052852224Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.074521302Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.074566583Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.094972657Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.095021518Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.115447143Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.115490344Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.134949331Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.135013852Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.154627713Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.154671133Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.175424634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.175473505Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.19536658Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.195411651Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.215625432Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.215663323Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.238705086Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.238763687Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.260800891Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.260844232Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.280933391Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.283324795Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.300871067Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.303687929Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.31957981Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.319619871Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.339293692Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.339332333Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.356960107Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45019"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.35713808Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"566d69e4-355d-4152-a334-2b4c2cc622f5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45019","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.358136058Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.358145748Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.375930205Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.375973576Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.395308731Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.395354912Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.413953233Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.416300446Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.43288611Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.432939221Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.433131685Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.433263137Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528042566Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528119748Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528131208Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528149338Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528156658Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528165578Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528175159Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528207599Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528218649Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528224439Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.52823748Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.52824473Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.52825348Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.52825928Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.52826471Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.52827402Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528279271Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528286751Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528292671Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:42.528298731Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"eef4865b-10b5-4d36-962a-4eb859f8a64b","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized)]","latency_us":497358,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:42.528533505Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"530.100206ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eef4865b-10b5-4d36-962a-4eb859f8a64b\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eef4865b-10b5-4d36-962a-4eb859f8a64b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.060066362Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.061168203Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.061185963Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.083583324Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.083668596Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.105759511Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.105816522Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.124324222Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.124386293Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.144592494Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.144655735Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.163809597Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.166309663Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.182021991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.182088992Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.20046062Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.200526021Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.219499089Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.219573901Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.238504928Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.238577199Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.256948056Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.259391041Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.275839873Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.278300248Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.294912064Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.294966085Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.313540195Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.313604547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.331526416Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.331591587Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.350038795Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.350105897Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.368883511Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.368942332Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.387294199Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.387367121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.40582966Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.40588073Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.424796708Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.424847779Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.443146225Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.443227146Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.443415379Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.443546012Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537341153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537427975Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537436305Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537450125Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537457965Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537482516Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537490556Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537497326Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537506186Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537514626Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537530216Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537538917Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537545617Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537560857Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537569557Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537576297Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537588898Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537596587Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537606348Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:43.53770092Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c2361c54-f007-43f2-ab8a-94383b41bf36","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized)]","latency_us":477219,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:43.537908573Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.522737s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c2361c54-f007-43f2-ab8a-94383b41bf36\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c2361c54-f007-43f2-ab8a-94383b41bf36\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.061657044Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.062297045Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.062308595Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.085083094Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.085146925Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.105406007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.105463358Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.124174111Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.124239263Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.14211Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.142174252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.159916717Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.159975208Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.179828263Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.179883384Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.197446396Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.197501207Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.216249421Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.216312282Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.233465337Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.233526469Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.252901484Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.252978856Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.273603604Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.273655245Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.291792578Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.291844859Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.310485871Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.310536962Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.328256617Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.328329059Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.346754367Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.346807158Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.36381839Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.363875971Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.382356121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.382421592Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.40026985Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.400328991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.418574335Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.42100671Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.437559754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.437641565Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.437863329Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.438009282Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530442908Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.53051643Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.53052897Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.53055547Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530564031Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530569561Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530580961Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530586711Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530592051Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530598851Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530612591Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530624492Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530631592Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530641782Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530648732Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530655212Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530667003Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530672663Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530695713Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:45.530703283Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"2408882f-65c0-4dbe-a2c5-4eb5e2ffdc4d","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized)]","latency_us":468610,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:45.530939317Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.808955375s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2408882f-65c0-4dbe-a2c5-4eb5e2ffdc4d\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2408882f-65c0-4dbe-a2c5-4eb5e2ffdc4d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.340493454Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.341142006Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.341148876Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.362857445Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.362923046Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.382880662Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.382928043Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.401017795Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.401061066Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.418521206Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.418565187Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.435361566Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.435402926Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.452634583Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.452689614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.471878826Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.471922567Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.49115715Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.491206001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.509611569Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.509655659Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.528403903Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.528444334Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.546564836Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.546603917Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.564690719Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.56473067Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.583133038Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.583176868Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.600993626Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.601038516Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.619271881Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.619320462Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.638220519Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.63826685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.656681888Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.656740229Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.675177137Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.675222188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.693478183Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.693518004Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.711180818Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.711230389Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.711448193Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.711588936Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805331277Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805398618Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805410918Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805431528Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805440648Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805448969Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805457789Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805467109Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805474459Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805479379Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805490369Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805498949Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.80550604Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.80551453Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.80552173Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.80552848Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805555981Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805561701Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805566981Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:47.805649422Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d78dcffd-a6cb-4b01-992f-affddf82927f","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized)]","latency_us":464692,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:47.805912987Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.317946515s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d78dcffd-a6cb-4b01-992f-affddf82927f\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d78dcffd-a6cb-4b01-992f-affddf82927f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.125364017Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.12606546Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.12607926Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.148871939Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.14892516Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.1674712Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.167518951Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.188877143Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.188940954Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.207849351Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.207901782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.227257318Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.227333579Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.245511033Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.245563203Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.264655494Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.264708065Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.283062272Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.283112913Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.302316025Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.302381226Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.320473279Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.32053151Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.339106831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.339155262Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.357541069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.35759984Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.376040659Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.37610248Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.394916355Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.394967396Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.413819512Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.413872613Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.431945205Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.432000356Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.451902501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.451961862Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.470970001Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.471034573Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.489759186Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.489813457Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.507984941Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.508070982Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.508270756Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.508407399Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603469403Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603532654Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603539754Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603554614Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603559365Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603563265Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603567755Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603571315Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603586285Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603591205Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603595695Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603600025Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603605295Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603625226Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603630156Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603633616Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603638196Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603641776Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.603648536Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:49.603655166Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"55953226-244e-4ed9-829d-4ff1a2e5eec3","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized)]","latency_us":477811,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:49.60386509Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.494958579s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"55953226-244e-4ed9-829d-4ff1a2e5eec3\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"55953226-244e-4ed9-829d-4ff1a2e5eec3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.099829686Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.100448628Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.100462028Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.123232806Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.125684361Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.144054438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.144100579Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.165573163Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.165616324Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.184167644Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.184214815Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.202482281Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.202535922Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.223475436Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.223518287Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.243384671Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.243433372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.263585622Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.263631713Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.284511797Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.284570438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.303979104Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.304024195Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.32176712Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.321809841Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.341866279Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.3419255Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.35988948Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.359930051Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.37838027Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.378424601Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.39743535Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.39747615Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.415782136Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.415856908Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.43502708Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.435083021Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.453064451Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.453115492Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.471434218Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.471479299Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.48954849Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.489614091Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.489862766Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.490044419Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581411636Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581502618Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581521088Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581545189Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581554059Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581560639Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581566739Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581574369Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581582329Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581592109Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.58160017Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.58162582Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.58163407Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.5816399Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.58164627Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581653991Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581671351Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581677991Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581683851Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:52.581689831Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"5f6abae7-936c-450c-ad23-6aaa6de31290","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized)]","latency_us":481400,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:52.581932976Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.887194142s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5f6abae7-936c-450c-ad23-6aaa6de31290\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5f6abae7-936c-450c-ad23-6aaa6de31290\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.46979871Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.470593944Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.470610325Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.493961133Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.494052265Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.51610099Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.516166261Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.537558993Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.537607704Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.556718285Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.559062418Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.574781376Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.574847607Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.592905149Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.59296805Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.611228445Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.613163411Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.629370919Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.62946685Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.648298666Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.648354547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.667268264Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.667315855Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.686507697Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.686568508Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.705216531Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.705264252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.723743931Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.723802052Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.742174739Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.74223291Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.760845212Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.760904013Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.779411963Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.781757386Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.797288931Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.797345882Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.815901203Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.815957084Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.833601188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.833671069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.852422153Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.852487234Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.852687078Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.85281627Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946339087Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946396338Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946410958Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946416958Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946424139Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946430829Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946437469Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946442969Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946460339Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946466259Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946475969Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94648306Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94648872Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94649305Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94649745Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94650802Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94651365Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94651808Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.94652227Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:54.94653023Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"46614458-19a4-429a-9331-52267250efcf","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized)]","latency_us":476135,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:54.946738834Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.935711637s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"46614458-19a4-429a-9331-52267250efcf\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"46614458-19a4-429a-9331-52267250efcf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.883850304Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.884493576Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.884511786Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.90758113Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.90762992Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.925744953Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.925791404Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.944004898Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.944059869Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.962703681Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.962745422Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.979054331Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:58.979101402Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.000289551Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.000331382Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.019308601Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.019355511Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.038960651Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.039000322Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.059110621Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.059161192Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.0797824Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.079828401Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.099032934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.099079755Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.117486243Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.117528004Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.136172126Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.136226587Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.155869857Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.155914798Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.175729222Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.175770873Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.195527665Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.195566816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.214480833Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.214523494Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.234316727Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.234358058Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.253386397Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.253430548Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.271781495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.271829676Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.27205218Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.272185132Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.36849285Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368576652Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368584592Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368592342Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368597672Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368605292Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368631313Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368637753Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368650083Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368657923Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368664303Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368672054Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368677634Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368682134Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368696714Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368701424Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368705624Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368711264Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.368717304Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:12:59.368777696Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8f95c052-8543-4390-a612-bc7a11b56457","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized)]","latency_us":484506,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:12:59.36899558Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.131321457s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f95c052-8543-4390-a612-bc7a11b56457\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:12:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8f95c052-8543-4390-a612-bc7a11b56457\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.501639202Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.502696132Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.502707282Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.525330677Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.525395958Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.545470476Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.54783883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.565135297Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.565200489Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.583447644Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.583505534Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.601590457Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.60395757Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.619988194Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.622334787Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.638171698Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.638227239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.656112787Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.656172879Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.675036225Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.675089906Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.693654096Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.693721578Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.712568644Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.712637155Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.731095174Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.731155425Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.74995724Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.750018071Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.768427439Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.770892554Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.787185213Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.787260105Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.805605851Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.805656863Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.825835803Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.825886764Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.844834952Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.847248716Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.863452293Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.863508874Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.882124476Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.882207268Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.882395641Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.882532314Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974778427Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974843658Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974854458Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974882109Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974899239Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974905099Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97491193Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97492641Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97493187Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97493867Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97494665Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97495328Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97495986Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.9749663Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.97497198Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974977211Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974983931Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974990391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.974997251Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:02.975027372Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a04bed0d-8cb8-4a6b-9c33-9ffd085cea6b","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized)]","latency_us":472708,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:02.975314727Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.582838929s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a04bed0d-8cb8-4a6b-9c33-9ffd085cea6b\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a04bed0d-8cb8-4a6b-9c33-9ffd085cea6b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.559263577Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.560546571Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.560559081Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.581017337Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.581081868Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.597905227Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.597948918Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.618645078Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.618692208Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.636445244Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.636494245Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.65638447Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.656438991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.67379107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.673858041Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.692575695Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.692615065Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.709842002Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.709892062Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.728465214Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.728516255Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.746490864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.746549266Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.764197819Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.76424159Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.783379611Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.783425802Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.800958444Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.801017255Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.820859759Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.82090484Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.838358101Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.838400852Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.856730538Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.856774069Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.875514903Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.875560434Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.894014213Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.894064063Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.911077766Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.911138617Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.929103557Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.929159687Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.929364251Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:05.929489794Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020204139Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020310231Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020322071Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020330981Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020341141Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020351401Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020357482Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020367122Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020373422Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020379512Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020385532Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020393142Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020401422Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020409173Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020417043Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020425153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020436713Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020447173Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020453194Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:06.020497634Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"b0cd7462-4bf7-4628-a321-443d9a6811eb","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized)]","latency_us":460730,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:06.020748299Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"5.243829129s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b0cd7462-4bf7-4628-a321-443d9a6811eb\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b0cd7462-4bf7-4628-a321-443d9a6811eb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.256791264Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.258061317Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.258095638Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.279643464Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.282161561Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.298561543Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.298628564Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.317273647Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.317336558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.337111302Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.337181074Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.354979311Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.355042922Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.374035692Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.374113943Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.391391351Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.391455342Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.409897232Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.409952883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.428388682Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.428457274Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.448120785Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.450293645Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.467184816Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.467239848Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.486625454Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.486693726Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.505005633Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.507432147Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.523988061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.524040442Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.541661517Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.541723438Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.560389711Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.560457003Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.578007776Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.578067577Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.59944462Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.599508991Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.61899425Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.619053931Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.638255945Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.638322556Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.63853136Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.638658512Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732509919Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732575291Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732583911Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732590231Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732596721Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732609391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732615792Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732620292Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732628352Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732642112Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732649792Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732657312Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732663192Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732669282Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732684313Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732691843Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732697713Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732701913Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.732707563Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:11.732789705Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"0d52d9cd-802b-440d-be9f-8d44f6806a55","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized)]","latency_us":476477,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:11.733024579Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.723001087s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0d52d9cd-802b-440d-be9f-8d44f6806a55\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0d52d9cd-802b-440d-be9f-8d44f6806a55\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:12.377207316Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33987"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:12.377313608Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"99950c26-0e2c-4309-9fd3-170b09a38261","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33987","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":64,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.443865687Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.444476958Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.444489758Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.467751656Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.467809128Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.487995766Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.48805506Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.506802084Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.506847475Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.525126642Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.527524726Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.543836106Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.543883577Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.561738825Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.561790884Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.580305637Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.580361888Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.599167634Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.599225566Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.618024321Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.618075692Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.637319267Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.637384618Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.656142873Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.656200514Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.67449647Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.674547002Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.693007862Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.695426265Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.711455551Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.711504112Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.729474593Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.729521854Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.74886952Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.748917571Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.767010924Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.767068165Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.785183078Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.787578022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.803538576Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.803601837Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.821348114Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.821416635Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.821629009Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.821770072Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917323541Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917369961Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917378511Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917387672Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917393472Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917398762Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917403532Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917409702Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917427823Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917431672Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917443053Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917446703Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917450153Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917454643Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917458933Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917462723Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917466983Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917471323Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917475713Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:16.917522744Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"4abc9f74-3f6d-428a-a74f-99481398499e","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized)]","latency_us":474521,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:16.917740158Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.155992828s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4abc9f74-3f6d-428a-a74f-99481398499e\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4abc9f74-3f6d-428a-a74f-99481398499e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.063353062Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.064450902Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.064462233Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.086791053Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.086840254Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.1061591Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.106200471Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.125378973Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.125417824Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.143835334Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.143891715Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.163397814Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.163443954Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.181952165Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.182009856Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.200794592Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.200845643Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.219080788Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.219124016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.23818023Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.238231761Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.257123169Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.25717514Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.274879335Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.274931747Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.29304529Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.293102801Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.31203973Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.31208008Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.32943895Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.32948088Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.348337698Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.348393709Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.367348448Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.367398269Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.38594333Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.385992151Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.404699985Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.404756286Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.425338625Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.425406626Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.444346795Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.444394286Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.444582589Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.444716372Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.538889154Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.538960355Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.538972275Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.538981506Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.538988686Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.538997416Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539016827Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539032157Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539041027Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539055997Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539064717Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539071128Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539076377Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539083238Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539093678Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539100468Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539107048Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539115518Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539122158Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:21.539171749Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a11cce23-b444-4f76-a987-d0a7b36cab41","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized)]","latency_us":476344,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:21.539403274Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.767705211s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a11cce23-b444-4f76-a987-d0a7b36cab41\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a11cce23-b444-4f76-a987-d0a7b36cab41\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.295724768Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.296742547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.296751897Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.319622548Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.319698229Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.340101445Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.340175897Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.359959031Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.360031192Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.377900471Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.377968232Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.395094237Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.395154349Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.411794435Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.411871436Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.429509261Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.429571772Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.446179848Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.446244909Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.465146407Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.465200068Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.482770061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.482838445Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.501031997Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.501094588Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.517217495Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.517287016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.534227098Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.534297019Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.552929312Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.553007883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.571137417Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.571201978Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.589709889Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.58976856Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.607838503Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.607894774Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.625049769Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.625123531Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.642555772Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.644543978Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.65878193Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.658859572Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.659090186Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.659241979Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751231722Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751286073Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751295113Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751315653Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751327854Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751334984Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751339604Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751346054Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751350554Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751354794Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751360225Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751364654Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751371125Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751377065Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751382745Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751388175Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751392415Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751396685Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.751407675Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:26.751431696Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"9eecd145-5cae-405a-9bda-6c359b4b3241","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized)]","latency_us":456282,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:26.75166051Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"9.239754129s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9eecd145-5cae-405a-9bda-6c359b4b3241\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9eecd145-5cae-405a-9bda-6c359b4b3241\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:35.989863939Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:35.99046443Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:35.99047239Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.01390377Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.013956621Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.033541911Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.033606182Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.053050689Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.05309257Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.072936194Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.072993595Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.091833291Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.091882502Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.109543096Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.109608867Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.128139487Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.128182388Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.14570623Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.145757491Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.164381022Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.164434654Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.183797849Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.18386352Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.201656547Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.201705848Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.220073665Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.220116296Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.238124796Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.238166747Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.257070434Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.257113305Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.274637686Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.274685957Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.292642177Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.292682808Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.311209818Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.311270219Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.329769238Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.329808989Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.34785074Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.347890741Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.366134086Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.366190817Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.366395661Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.366534313Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.459824986Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.459928708Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.459951488Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.459962348Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.459973319Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.459983789Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.459991939Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.460000099Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.460007169Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.460013269Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.46002258Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.46003293Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.46004099Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.46004871Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.46005669Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.46006476Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.460085091Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.460095421Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.460103131Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:36.460112561Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d990606a-d337-45a2-8c2f-aa1842e06da8","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized)]","latency_us":469827,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:36.460363086Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"14.331735569s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d990606a-d337-45a2-8c2f-aa1842e06da8\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d990606a-d337-45a2-8c2f-aa1842e06da8\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:42.369329461Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61510"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:42.369409412Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"131d8a9d-5c14-4029-af0a-e14e1bb7e368","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61510","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":50,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.793693088Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.794688806Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.794698726Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.816344653Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.816417705Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.836274169Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.836347491Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.855144016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.855204087Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.874682844Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.874748525Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.892800297Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.892866878Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.911514251Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.911574252Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.93000478Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.930066831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.948379357Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.948444348Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.967645701Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.967712142Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.98664023Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:50.986698541Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.005324322Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.005385944Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.027492929Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.027556011Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.047297213Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.047362194Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.065531608Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.065587849Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.085315191Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.085406903Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.104179317Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.104253358Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.123599014Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.123654565Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.142232496Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.142295007Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.161967758Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.162027059Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.180889935Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.180965867Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.181297343Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.181459386Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279277291Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279359073Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279380773Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279391684Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279402314Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279412344Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279423154Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279435454Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279443465Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279451435Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279457685Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279483855Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279494735Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279500566Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279508626Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279516446Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279523786Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279531846Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279537756Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:13:51.279548356Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"24905c2c-2c8e-4d08-a968-8e0346e45e12","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized)]","latency_us":485049,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:13:51.279804931Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"9.299256109s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"24905c2c-2c8e-4d08-a968-8e0346e45e12\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:13:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"24905c2c-2c8e-4d08-a968-8e0346e45e12\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.58082523Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.581550534Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.581568704Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.604186769Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.60425258Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.624642925Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.624705396Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.644443188Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.644521499Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.664805132Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.664880883Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.684611245Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.684684697Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.70284219Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.702907191Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.721971461Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.722033232Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.740053353Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.740133534Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.758996611Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.759062322Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.777635813Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.777698054Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.796669782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.799171138Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.815304244Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.815375176Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.833781253Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.836221888Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.852013308Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.85207982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.870731412Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.870789383Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.889184681Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.889254342Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.90766121Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.907736321Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.926314902Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.926383884Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.944886543Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.944955614Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.963315541Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.963398893Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.963638637Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:00.96377571Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057304407Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057353778Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057363298Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057370198Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057376218Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057382938Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057389538Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057395018Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057420279Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057425409Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057429679Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057439269Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057445269Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.05745099Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.05745671Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.05746118Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.05746665Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.05747106Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.05748147Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:01.057513511Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c7f4d6ae-ec64-4308-8c4a-9e9060af82b0","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\", TimeNotSynchronized), (\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\", TimeNotSynchronized), (\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\", TimeNotSynchronized), (\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\", TimeNotSynchronized), (\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\", TimeNotSynchronized), (\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\", TimeNotSynchronized), (\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\", TimeNotSynchronized), (\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\", TimeNotSynchronized), (\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\", TimeNotSynchronized), (\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\", TimeNotSynchronized), (\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\", TimeNotSynchronized), (\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\", TimeNotSynchronized), (\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\", TimeNotSynchronized), (\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\", TimeNotSynchronized), (\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\", TimeNotSynchronized), (\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\", TimeNotSynchronized), (\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\", TimeNotSynchronized), (\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\", TimeNotSynchronized)]","latency_us":476195,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:01.057767615Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"21.628564374s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c7f4d6ae-ec64-4308-8c4a-9e9060af82b0\", \"content-length\": \"124\", \"date\": \"Fri, 09 Feb 2024 21:14:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c7f4d6ae-ec64-4308-8c4a-9e9060af82b0\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:12.399962735Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:12.400030746Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"68d91876-f7dc-4d4b-b39b-0441c8e87324","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55810","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e86c47c-f102-4dae-b3de-66cf774453ac, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 56b50ae5-31a2-4588-b795-6af274c2aaff, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 71d25d50-2cba-4615-b456-83b65add8d14, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 14d24551-1792-4570-9855-8dc6e91f2988, kind: Service { id: 71d25d50-2cba-4615-b456-83b65add8d14 }, name: Name(\"ntp-71d25d50-2cba-4615-b456-83b65add8d14\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 174, 184])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 9822dded-9084-4bcb-acff-e08ee8809728, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ef72eb88-de65-4f60-8270-28e9b6c1431f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 93bdd1d6-923f-4556-a2e7-c8932e375777, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 50e50144-887b-4bfb-9c64-0dab98e559b3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 26fa6230-6a52-4867-a073-2918d92baa6b, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 2130b665-8c04-446a-ae31-3ccbd36963c1, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba0d150e-a49b-428d-a98d-600e9c760fb3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b9446a8d-d093-495b-8393-dd3553e581a3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 011955b8-afdc-40b4-8778-e8e8a097f069, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ab9d3036-e56a-4616-919f-3d0e8c6a163e, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 62da62bb-f12c-4c87-939f-211eedf441e6, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 349d1df8-3bee-48ae-8910-2ff7ce874216, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 6c7363af-c204-4d04-b820-2c48b938b961, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: b8137b72-6acd-4a71-986f-cc0c322220c4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 57b93370-4580-4b14-a531-d02ceda72d24, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: cfd21b45-159e-4d6c-8882-a1352c6429cd, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 0d428723-26c3-488a-bda3-7f6e0f53d758, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cb06eb84-0695-450c-8917-eec239f61fb4, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.684846259Z","hostname":"sock","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.68593737Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0e86c47c-f102-4dae-b3de-66cf774453ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.68595405Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.706990516Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 10efe7b5-4c14-431b-ae96-fd6d968a40df, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.707039637Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.728369178Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56b50ae5-31a2-4588-b795-6af274c2aaff, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.728414589Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.746174765Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9822dded-9084-4bcb-acff-e08ee8809728, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.746229486Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.76495694Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ef72eb88-de65-4f60-8270-28e9b6c1431f, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.765015301Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.784427558Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 93bdd1d6-923f-4556-a2e7-c8932e375777, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.784467298Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.802582911Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 50e50144-887b-4bfb-9c64-0dab98e559b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.802626282Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.820668473Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0bcc78b8-eeff-44dc-a9db-f18fef587e2c, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.820709954Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.839887196Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 26fa6230-6a52-4867-a073-2918d92baa6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.839946897Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.858169691Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2130b665-8c04-446a-ae31-3ccbd36963c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.858209852Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.876781973Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2d87007f-dd78-4a53-9e6a-affe1e8578dd, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.876832874Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.895262202Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c7363af-c204-4d04-b820-2c48b938b961, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.895311273Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.912862395Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.912898736Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.930441398Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: b8137b72-6acd-4a71-986f-cc0c322220c4, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.930490179Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.949110681Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 57b93370-4580-4b14-a531-d02ceda72d24, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.949169892Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.967664061Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 35709a05-cf4a-4b4c-9fc9-3daafac7d776, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.967702542Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.986453236Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4e405e6c-5684-46dd-aa53-1ccf5757accd, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:22.986502157Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.00461338Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cfd21b45-159e-4d6c-8882-a1352c6429cd, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.00465288Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.025140237Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0d428723-26c3-488a-bda3-7f6e0f53d758, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.025188107Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.043166237Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cb06eb84-0695-450c-8917-eec239f61fb4, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.043205518Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.061414832Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.061460613Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.061721908Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.06184699Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.172156145Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.17239012Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.172436501Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.173683623Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.188647118Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.188794421Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.188811341Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.188824831Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.202284218Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.20457193Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.377634407Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.377921943Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.377986754Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.377998094Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.378030425Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.420652397Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.420972873Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.421021934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.421033934Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.421110366Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.425639679Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.43006077Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.43444295Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.43879629Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.443659769Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.448544449Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.586125744Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.590538235Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.595121409Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.599266125Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.603939211Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.608650188Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.61367243Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.61858508Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.623185014Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.62784376Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.633448923Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.639684347Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.646020694Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.651855561Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.658513233Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.664505393Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.670862359Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.676138636Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.681689608Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.687555056Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.693509345Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.739679083Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.791173238Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.822144517Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.88985517Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.925666287Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.927429909Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.92744143Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.984826803Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", _ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 144, 141]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.984891814Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.988760545Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:23.988784436Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.043719084Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", _ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 194, 87]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.043796916Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.0472804Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.04729429Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.101358932Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", _ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 248, 94]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.101426523Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.104295206Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.104310056Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.159116013Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", _ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 134, 169]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.159179174Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.16116178Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.16117404Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.218850229Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", _ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 219, 17]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.21891246Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220165273Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220182474Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.22053231Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.22056383Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220700893Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220712203Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220795645Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220811165Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220875636Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.220888527Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.221503178Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.224057655Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.226946228Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.226965788Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.226990388Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.22926299Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.231573843Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.231597713Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.231609343Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.233552929Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.365546662Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.372616452Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.380590668Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.387412653Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.393540096Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.400583035Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.406631886Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.41337198Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.419272268Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:24.427066591Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.071677641Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.071920775Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.116279979Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.116519864Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.136940129Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.137190243Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.475287889Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:26.475503903Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 29.721297ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:26.645752288Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 52.120395ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.438743525Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.439631912Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.439943958Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 67.748255ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.466031997Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 55.306238ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.481993949Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 245.658984ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.522661956Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.530756994Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.531345745Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 113.01273ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.57412541Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 95.809086ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.686905071Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.688384098Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.689405237Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 211.527085ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.799927435Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 253.611639ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.849411024Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 578.181356ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.876032343Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.876247696Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.876257607Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.893072425Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.893986342Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.896225553Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.896292044Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 70.452253ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:27.92820714Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.928510866Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.928609588Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.940131919Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.952123209Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:27.981433897Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 127.851517ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.065762044Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 562.170006ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.253305097Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 596.570461ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.271317028Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:37.387386739Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 726.274758ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.40868721Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 52.287039ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.426497667Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 61.352584ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.447411341Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 32.858909ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.458513534Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 62.262732ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.470574136Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 126.873259ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.488992013Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 105.996202ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.500196379Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 56.849402ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.510289834Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 79.125722ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.52421044Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 67.554284ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.545526501Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 117.155159ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.582984458Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 25.9772ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.596567068Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 52.345987ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.607487818Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 285.75715ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.620871104Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 277.872255ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.647297019Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 70.72818ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.660506872Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 254.482503ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.673229515Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 494.294574ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.687736152Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 116.92797ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.697926769Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 105.95932ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.708420311Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 494.799703ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.718604148Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 100.063687ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.745711026Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 1.133931199s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.89954866Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 438.731459ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.913391814Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 71.033715ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.921533653Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 248.011304ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.931111499Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 149.448479ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.94039691Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 225.520519ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.951255439Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 351.035903ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.963540384Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 267.595203ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.976005853Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 265.005614ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:37.987327881Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 84.939051ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.009872345Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 61.927325ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.028444906Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 533.660201ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.101841614Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 118.872599ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.116460682Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 71.537474ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.129319968Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 793.129022ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.14847293Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 596.624326ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.190388979Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 1.135284027s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.203763674Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 692.813013ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.215880517Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 197.703343ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.228450498Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 867.599839ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.242158009Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 311.902493ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.254453435Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 775.422714ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.267765099Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 203.36332ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.279041526Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.111286475s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.327631988Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 949.410852ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.363089789Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 204.919988ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.439035944Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 1.12080211s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.49550318Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.031670996s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.581463218Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 853.548869ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.648955517Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 527.382007ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.662498015Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.149358441s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.80114712Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:38.860934388Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:38.86867097Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 1.406213996s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.923317043Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 904.813016ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:38.956035274Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 1.127503165s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.048295267Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 760.64665ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.062336655Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.172768152s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.123423376Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 1.201172858s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.206277997Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 74.757923ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.217591345Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 65.616553ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.226686942Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 91.592333ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.30555185Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 62.357246ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.31647001Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 1.057090054s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.328498141Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 855.426986ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.351661246Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 154.75499ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.38947302Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:39.389806217Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 191.510994ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.412009204Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:39.412433022Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.123113083s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.465593038Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 988.944905ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.769924604Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.304806338s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.814753127Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 401.760782ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.877615931Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 1.156009989s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.916132628Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 420.880523ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:39.957511447Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 715.650103ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.000528687Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 573.04081ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.048129471Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.464245478s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.096757843Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 28.516012ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.155690395Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 777.761972ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.346155272Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:40.346523468Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 67.131609ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.370461438Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 1.453234153s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.383699941Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 71.183548ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.395192312Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 1.36386997s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.409399383Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 796.306099ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.42122057Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 659.299328ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.433536626Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 730.808447ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.450694431Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 890.371982ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.464760259Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 116.726418ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.479234754Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 577.129503ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.495988491Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 172.525901ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.513432912Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 781.804615ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.602468986Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 100.678208ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.616719898Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 582.587736ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.634528695Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 49.95442ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.647062745Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 240.965186ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.700368423Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 101.626142ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.711778943Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.473259803s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.729234893Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 291.399464ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.742566948Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 590.452445ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.771269535Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 293.93133ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.826145042Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 763.896657ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:40.955765792Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 418.764245ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.048885001Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 922.124801ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.117064373Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 735.101075ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.128199587Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 603.074376ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.140432512Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 1.144384616s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.152614556Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 526.773721ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.170366281Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 635.417599ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.182493774Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 690.087495ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.196097094Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 808.691434ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.229959915Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 677.957299ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.2416756Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 1.146782578s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.368179313Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.331215007s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.378565193Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 899.887355ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.395647657Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 781.54863ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.480417372Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 806.298856ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.574961088Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.088913915s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.710280222Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 1.151468983s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.731507882Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 740.231131ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.754400592Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 1.047489306s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.785028584Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 658.131285ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.83220007Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:41.833714328Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:41.834110505Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 711.735689ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.852276679Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 667.331025ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.877750376Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.114101932s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.900326451Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 516.017474ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:41.933079222Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 816.337579ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.053660956Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 859.920517ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.066367429Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 51.752743ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.077823449Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 113.240819ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.14215013Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 571.768538ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.211637036Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 190.290871ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.264604118Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 1.372429426s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.27290182Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.177489539s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.304120853Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 1.226838976s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.311849345Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.020602305s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.391158731Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 678.161927ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.460164088Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:42.464182882Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44163"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 544.920011ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.470948555Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:42.486356178Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"6e0d7697-9c29-4ef2-b92e-5a5bc836c9d7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44163","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":21630,"response_code":"200"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.497835854s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.499622711Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 931.928295ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.507865803Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 608.852182ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.523933857Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 626.547698ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.559182905Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 942.103519ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.577231296Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.18625821s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.722953321Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 818.510019ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.793070048Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.283342786s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.81168722Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 884.88019ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.842996575Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 609.49552ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.883129451Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 931.016046ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.896321064Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 1.165628615s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:42.939659119Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.269325732s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.027745766Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 788.057223ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.040069063Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 572.237171ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.146109149Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 1.019090645s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.156894897Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 1.421170273s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.198019542Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.288096104s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.424292506Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 955.763201ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.453596524Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 657.294333ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.493435584Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 1.161619736s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.504589869Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 691.555001ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.530346782Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 1.498814765s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.550591274Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.024470962s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.622653996Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 974.618096ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.656852504Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 1.015677294s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.728557341Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 872.079689ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.738970672Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 1.179404486s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.838617461Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 520.769249ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.849058093Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 524.606597ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:43.919245641Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.204464555s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.008934617Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.468042139s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.105001571Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 627.57349ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.115404512Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.261125288s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.162438206Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 743.295709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.187668918Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 585.269925ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.232760116Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 531.58345ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.307325865Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 1.187306198s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.378684035Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 630.120157ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.417751402Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.091485541s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.452923818Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 1.283749028s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.621160086Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 812.149136ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.62902373Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 1.097757679s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.639501572Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 842.77026ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.654963716Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 1.053759327s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.673966885Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 826.150096ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.720343436Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 1.082826916s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.753260341Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 555.714342ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.760867271Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 1.189105898s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.82950982Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.048019467s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.84747933Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 1.13299672s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:44.940326855Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 1.235215896s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.027042447Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 1.272027521s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.057584957Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 540.68509ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.065265118Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 899.669174ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.224790387Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 1.299322346s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.328430849Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.283243236s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.436000844Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 1.256279579s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.451890616Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.483872709s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.507509666Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.428323262s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.553958459Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 554.833183ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.562947874Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 1.091633681s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.574594688Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 637.672637ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.580974765Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 1.127419292s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.615081031Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 970.043441ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.736312436Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 1.191553369s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.745359392Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 764.066654ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.84435782Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.313239577s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.904140547Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 1.355211867s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:45.913795464Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 747.095359ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.031297861Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 533.691221ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.08513558Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 654.804055ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.129204809Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.126401526s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.139476217Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.070623698s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.230961217Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 564.204955ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.273582339Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 702.308542ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.341125469Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 635.002971ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.618957258Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 687.095004ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.628089786Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 1.206040313s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.6381407Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 526.029874ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.678264587Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 957.076296ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.719463163Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 1.212890492s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.73073441Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.221211336s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.742764721Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 613.127449ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.75248975Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 961.816745ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.791422854Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 713.474195ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.804506825Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 1.178796363s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.857135931Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 943.235048ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.947212174Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.210821604s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:46.99547188Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 503.063629ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.007921859Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 584.015627ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.055074694Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 500.435617ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.215199724Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.167794722s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.227963398Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 1.460356247s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.268613694Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 854.244368ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.280566354Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 921.66318ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.29016286Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.393478695s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.312764375Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 505.644808ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.327354593Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 865.219466ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.38061727Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.069212803s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.52552153Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.096885886s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.538848694Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 833.312097ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.652519791Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 952.996083ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.688465361Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 1.466952822s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.731219356Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 1.365455499s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.767760777Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 827.670676ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.850279082Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 1.386162732s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.878829006Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 625.695431ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.921090162Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 951.752916ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.958904806Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 770.891611ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:47.977591929Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 1.087095385s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.053653445Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 632.272731ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.14925356Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.384810787s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.225637432Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 643.637618ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.245466236Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 1.133339959s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.262991918Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.369065609s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.41183592Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 844.260462ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.50014222Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 1.307894401s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.560480978Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.062729744s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.606182097Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.163336035s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.651219584Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 1.499804069s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.667013304Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 793.927175ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.695206171Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.160228208s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.719350305Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 775.928193ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.748037731Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.145106332s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.760708174Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.271566293s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.794552145Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 1.468871435s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.909252891Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 1.148710269s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:48.936798586Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 1.032049713s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.148464682Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 542.268529ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.161716455Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 1.247842649s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.209274368Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 944.170109ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.279369165Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 1.320855621s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.353326543Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 565.781384ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.409172388Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 1.313066508s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.499423604Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 892.46147ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.534968956Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 954.892666ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.619582829Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.463977638s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.680366845Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 982.748764ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.714180276Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 571.056369ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.794340368Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.082002378s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.827917554Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 880.604062ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.881004379Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.363828949s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.890155656Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 567.43977ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.917306075Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 538.166074ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:49.98732726Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 614.763154ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.076762702Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 919.49033ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.095871273Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 566.912602ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.181062077Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 555.343702ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.194532094Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 1.41706941s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.238079294Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 927.840506ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.377302649Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 509.178828ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.389528974Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 648.923867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.440171564Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 1.287928854s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.467508915Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 589.444947ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.49391693Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 1.43388052s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.535210957Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 901.167309ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.583670757Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 833.652803ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.686956343Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.118256489s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.717669937Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 1.497648037s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.74287134Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 933.901783ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.762971709Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 560.350917ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.775098571Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 667.946186ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.789494416Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 1.3145277s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.827758738Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 605.368176ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.912808159Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.284650552s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:50.925138295Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 634.831566ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.02781726Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.225680226s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.098412827Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 1.247648006s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.115611882Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.243054552s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.156648985Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 812.796678ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.266629675Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 1.16738138s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.317760223Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 737.700444ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.349493666Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 735.713465ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.473171465Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.023565174s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.499898436Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 1.485281097s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.534541082Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 823.474648ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.548705482Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 550.35709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.672527785Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 715.362774ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.688229093Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 654.762897ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.710189986Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 609.79614ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.770831969Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069\") failed: Property not found. retry in 1.434807382s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.849723338Z","hostname":"sock","pid":1671,"zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 544.026967ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:51.988795671Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 722.240454ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.096773953Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3\") failed: Property not found. retry in 1.029855929s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.113772365Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 740.200085ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.157685121Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6\") failed: Property not found. retry in 995.156738ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.219473645Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961\") failed: Property not found. retry in 1.404254885s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.228602343Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b\") failed: Property not found. retry in 1.066600465s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.236984557Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560\") failed: Property not found. retry in 1.364818606s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.250896152Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 692.037467ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.333143842Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 534.460196ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.373077785Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216\") failed: Property not found. retry in 1.333346205s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.382125421Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1\") failed: Property not found. retry in 806.412386ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.393136683Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd\") failed: Property not found. retry in 1.13172041s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.402179179Z","hostname":"sock","pid":1671,"zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd\") failed: Property not found. retry in 1.405519084s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.411254136Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e\") failed: Property not found. retry in 1.050738008s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.420243281Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4\") failed: Property not found. retry in 844.765012ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.494951051Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf\") failed: Property not found. retry in 1.303809221s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.534770173Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776\") failed: Property not found. retry in 912.049975ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.543806938Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3\") failed: Property not found. retry in 1.0583546s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.830833207Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4\") failed: Property not found. retry in 1.253887199s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.908499973Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758\") failed: Property not found. retry in 631.986185ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:52.917770863Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24\") failed: Property not found. retry in 567.4149ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:53.03088931Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd\") failed: Property not found. retry in 1.190674772s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:14:53.04015946Z","hostname":"sock","pid":1671,"zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::c exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:53.198476166Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:53.198532507Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::c exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:53.754945911Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:14:53.754999412Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:05.400072226Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:05.400115907Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:11.824459726Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:11.824525267Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:12.382789815Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:12.382853926Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.388817966Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.049092062s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.389906436Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.059374921s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.390034728Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.059956042s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.390332234Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.059155987s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.390601028Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.058891372s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:12.517372825Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:12.517722121Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"8f183342-acd9-48f9-b4c3-c7ab0706ef28","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49408","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.695866452Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:710","total duration":"225.186148047s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.696256629Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"16.907203524s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.726055866Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.396795585s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.726446493Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.395648903s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.726603316Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.395690104s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.72680506Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.396169502s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.726884691Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.399172278s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.727460492Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.397852103s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.727554053Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.395745325s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.727784537Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.396759133s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.727844999Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.398107778s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.72791061Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.400410651s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.727995521Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.397322394s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.728056753Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.397852074s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.728136874Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.396581821s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.728192985Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.39871435s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.728429049Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"235.39712052s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.72849153Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.398542387s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:12.728560112Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"235.403272594s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:13.342457871Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:13.342486032Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:17.454536754Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"Ensuring address fd00:1122:3344:101::a exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:19.275611275Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::a, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:19.275694946Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:19.827171932Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"242.496185671s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:19.827311191Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"242.494625172s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:19.827461272Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"242.494978089s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:19.827541244Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"242.497009605s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:19.827624495Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"242.496025188s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::a exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:19.831739871Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:19.831753461Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:19.896815925Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"242.570557857s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:19.936627836Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"242.605702431s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:20.19510897Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:20.195175511Z","hostname":"sock","pid":1671,"zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:20.347081399Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"243.014315112s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:20.446910968Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.115629552s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:20.556873508Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.226293283s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:20.846958038Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:710","total duration":"233.336234351s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:20.886815214Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"243.554620439s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:20.936942204Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.60641804s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:20.938653015Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"243.609990316s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:21.086855765Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"243.754499179s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:21.28709441Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"243.955474197s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:21.386911312Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"244.054869193s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:21.516945427Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"244.187935854s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:21.607062281Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"244.275993221s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:22.980399716Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"245.648135948s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:22.987224091Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.656413941s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:23.288017371Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"245.956397597s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:23.349809965Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"246.016584282s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:23.437301191Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"246.104315212s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:23.537594661Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"246.204947958s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:23.847619581Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"246.515306385s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:24.511791429Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"247.179793633s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:24.750119903Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"247.420646274s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:24.816199216Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:24.818167392Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:710","total duration":"237.306637341s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:24.882557974Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"247.550810954s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:25.108560012Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"247.781252924s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:25.168261528Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"247.835873346s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:25.728775287Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"248.399035283s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:25.984687654Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"248.653323091s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:26.056633155Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"248.72513509s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:26.625266853Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"249.292072866s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::9 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:26.657565986Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:26.657622887Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:26.809447864Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"249.476896238s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:27.075963346Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"249.744727256s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:27.091279077Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"249.759664569s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:27.09852217Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"249.764810114s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/e8553772-9db2-44b1-acc1-a08a65e40999/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::9 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:27.217716288Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:27.21779501Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up external-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:27.507870974Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2084"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:27.507931125Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:27.759780598Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"250.426705943s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f15ee456-96db-49cb-87fc-8bc73db28bb8\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:27 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"f15ee456-96db-49cb-87fc-8bc73db28bb8\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:28.01046139Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"250.678467915s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"33a65dfd-fa25-40bd-b31e-447d10fd5fa4\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:27 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"e8553772-9db2-44b1-acc1-a08a65e40999\\\\\\\"\\\", request_id: \\\"33a65dfd-fa25-40bd-b31e-447d10fd5fa4\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:28.22783761Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"250.895595211s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"a880e363-6e35-411b-9653-6b4b163ff8cf\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:28 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"e8553772-9db2-44b1-acc1-a08a65e40999\\\\\\\"\\\", request_id: \\\"a880e363-6e35-411b-9653-6b4b163ff8cf\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:28.549733018Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"251.216154584s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"4fcbb4f1-d244-4843-9378-2f9a25d60191\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:27 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"4fcbb4f1-d244-4843-9378-2f9a25d60191\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:28.622393272Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"251.288564093s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"d605d92c-1f60-4e10-aa7e-678e75a8e87e\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:28 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"d605d92c-1f60-4e10-aa7e-678e75a8e87e\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:28.988678506Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"251.655871696s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"eba323dc-965d-499c-ab2d-8ed9dd47799d\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:28 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"eba323dc-965d-499c-ab2d-8ed9dd47799d\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:29.28742039Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"251.954480487s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"6f571931-094e-4c17-9c00-3085604667a3\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:29 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"6f571931-094e-4c17-9c00-3085604667a3\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:29.633245927Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"252.305929909s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"22dc7006-f129-44d5-96b4-858ec77732a2\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:29 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"e8553772-9db2-44b1-acc1-a08a65e40999\\\\\\\"\\\", request_id: \\\"22dc7006-f129-44d5-96b4-858ec77732a2\\\" }\""} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:29.652813536Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1168","retry_after":"21.488247501s","error":"\"Metric registration error: Error Response: status: 503 Service Unavailable; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"564ed726-ed2e-40a5-87f5-d438fdf877a6\\\", \\\"content-length\\\": \\\"133\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:29 GMT\\\"}; value: Error { error_code: Some(\\\"ServiceNotAvailable\\\"), message: \\\"Service Unavailable\\\", request_id: \\\"564ed726-ed2e-40a5-87f5-d438fdf877a6\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:29.829799725Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.496474696s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"b69342dc-0ae6-4f00-a308-5b599b8a7433\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:28 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"b69342dc-0ae6-4f00-a308-5b599b8a7433\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:30.05939129Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.727003138s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"5f14c730-9e6d-4946-9473-9b057f3da48a\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:29 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"5f14c730-9e6d-4946-9473-9b057f3da48a\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:30.079375327Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"252.746693109s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"300c44d6-736c-4b2d-a09f-2aff106d75a5\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:30 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"300c44d6-736c-4b2d-a09f-2aff106d75a5\\\" }\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:30.997853246Z","hostname":"sock","pid":1671,"sled_id":"e8553772-9db2-44b1-acc1-a08a65e40999","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:31.192462979Z","hostname":"sock","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/sled_agent.rs:667","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"sock\" }","id":"e8553772-9db2-44b1-acc1-a08a65e40999"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:31.775275647Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"254.44209997s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"cca1e09d-ced5-4d34-a5ae-5fad374bd101\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:31 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"cca1e09d-ced5-4d34-a5ae-5fad374bd101\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:32.239189572Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"254.906646077s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0a0f7907-3d4c-443c-adfb-055925841765\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:32 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"0a0f7907-3d4c-443c-adfb-055925841765\\\" }\""} {"msg":"Ensuring address fd00:1122:3344:101::8 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:33.736893214Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:33.736967216Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:33.756738028Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Ensuring address fd00:1122:3344:101::8 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:34.297274431Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:34.299894879Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up external-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:34.574249134Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2084"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:34.574310506Z","hostname":"sock","pid":1671,"zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:34.988895106Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"257.655061397s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"761a7966-2df0-4656-b44b-9e8cc18b2f21\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:34 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"761a7966-2df0-4656-b44b-9e8cc18b2f21\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:35.804149281Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:35.804184682Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:35.804190231Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:35.804195512Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:35.804208792Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:15:36.543720876Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"259.210544689s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"adff9e80-846b-48df-970d-6fa58f52c0df\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 09 Feb 2024 21:15:36 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"adff9e80-846b-48df-970d-6fa58f52c0df\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:36.607195461Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:36.607221862Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:36.810087556Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:36.810105526Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: e8553772-9db2-44b1-acc1-a08a65e40999, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:36.810112176Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:37.566130393Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:37.566170764Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289073636Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_cfd21b45-159e-4d6c-8882-a1352c6429cd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289145928Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_35709a05-cf4a-4b4c-9fc9-3daafac7d776"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289155278Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_ba0d150e-a49b-428d-a98d-600e9c760fb3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289170338Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_b9446a8d-d093-495b-8393-dd3553e581a3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289177288Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_clickhouse_2d87007f-dd78-4a53-9e6a-affe1e8578dd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289184838Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_62da62bb-f12c-4c87-939f-211eedf441e6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289191269Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_ab9d3036-e56a-4616-919f-3d0e8c6a163e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289198549Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_5505a386-30f2-436c-a5d3-2bc80bc9d560"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289205209Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_oximeter_011955b8-afdc-40b4-8778-e8e8a097f069"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289218049Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_349d1df8-3bee-48ae-8910-2ff7ce874216"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289223049Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_57b93370-4580-4b14-a531-d02ceda72d24"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289230809Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_54ecaabe-a3b7-47b1-aa6a-55c26a1ea0bf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289242899Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_cb06eb84-0695-450c-8917-eec239f61fb4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.2892507Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_6c7363af-c204-4d04-b820-2c48b938b961"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.28926349Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_external_dns_2130b665-8c04-446a-ae31-3ccbd36963c1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.28926835Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_4e405e6c-5684-46dd-aa53-1ccf5757accd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.28927287Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_0d428723-26c3-488a-bda3-7f6e0f53d758"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289526095Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_b8137b72-6acd-4a71-986f-cc0c322220c4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.289534795Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_external_dns_26fa6230-6a52-4867-a073-2918d92baa6b"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.28982476Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.293333525Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.296995272Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"ffb394f9-2b08-4dcf-90bb-a110754228c6","remote_addr":"[fd00:1122:3344:101::1]:49975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":77610926,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.310154373Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.310391268Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.310661443Z","hostname":"sock","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.315194436Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.315772617Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.362664297Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/56f2c6d3-9416-4aae-9f83-77cafa4b4cdb/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.365707813Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/56f2c6d3-9416-4aae-9f83-77cafa4b4cdb/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.504102593Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/56f2c6d3-9416-4aae-9f83-77cafa4b4cdb/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.618453662Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.840815884Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.842878882Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"1c3287e5-e009-4e32-9f21-0926401041a5","remote_addr":"[fd00:1122:3344:101::b]:52935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1496,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.846382246Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.846407657Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.846873445Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.847234032Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"cb7e5767-20db-413e-b70e-9415fa8e1019","remote_addr":"[fd00:1122:3344:101::b]:52935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3992,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.866800051Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:49638"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.866928933Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"69a22b41-bd98-49b0-8d7e-d2249084aa52","remote_addr":"[fd00:1122:3344:101::a]:49638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.868143016Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.868155546Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.868494332Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:40.869281986Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"a032cee8-f0ac-4a31-bdea-c1700a8911cb","remote_addr":"[fd00:1122:3344:101::a]:49638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":2121,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba74ada2-b75b-44de-ba19-096762267f82\", \"content-length\": \"174\", \"date\": \"Fri, 09 Feb 2024 21:15:41 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"31bd71cd-4736-4a12-a387-9b74b050396f\\\"\", request_id: \"ba74ada2-b75b-44de-ba19-096762267f82\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.209246067Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.268668228Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.268692758Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.268698338Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.268717979Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.354555134Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.356181904Z","hostname":"sock","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"d9a4e2dd-d842-47ad-b850-5c51b2db0362","remote_addr":"[fd00:1122:3344:101::a]:63911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1592,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.369459368Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48077"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.36954287Z","hostname":"sock","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"8ebed10c-fe70-489f-8f16-42a9d72e1a21","remote_addr":"[fd00:1122:3344:101::b]:48077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.401669559Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.401731131Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"ea714797-356d-4c15-8096-1272c996768b","remote_addr":"[fd00:1122:3344:101::a]:56242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":23,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.401880963Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.401901294Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.402090667Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.402294651Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"dc36bef1-284b-4e98-8b9b-c2cc39c5dd9f","remote_addr":"[fd00:1122:3344:101::a]:56242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":439,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.406549849Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.40660078Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"89657b71-23ca-451e-b643-6f80b24d3fce","remote_addr":"[fd00:1122:3344:101::b]:59753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.407322603Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.407336663Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.407522657Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.40771459Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"8881bfb0-b0bc-44ae-8f84-7b1a86a7fe26","remote_addr":"[fd00:1122:3344:101::b]:59753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1002,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e07af2f2-36ad-4c1e-a826-baf9e11ee0f9\", \"content-length\": \"174\", \"date\": \"Fri, 09 Feb 2024 21:15:41 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"e07af2f2-36ad-4c1e-a826-baf9e11ee0f9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:41.818648203Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4f11a2a-6641-402d-a9cc-23af0035199f\", \"content-length\": \"174\", \"date\": \"Fri, 09 Feb 2024 21:15:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"d4f11a2a-6641-402d-a9cc-23af0035199f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:42.323730005Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:42.551975704Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34543"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:42.552034035Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"172116f2-04a0-42b5-8701-6768045a92ee","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34543","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":43,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fc4f4f46-c189-4ef8-9d7a-24a53ed1217d\", \"content-length\": \"174\", \"date\": \"Fri, 09 Feb 2024 21:15:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"fc4f4f46-c189-4ef8-9d7a-24a53ed1217d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:42.939117041Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e4b4dc06-b8d3-46ac-bed8-2a7661fe0d34\", \"content-length\": \"174\", \"date\": \"Fri, 09 Feb 2024 21:15:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"e4b4dc06-b8d3-46ac-bed8-2a7661fe0d34\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:43.562711107Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9472bbea-7078-48ce-aa17-4c8cf275af44\", \"content-length\": \"174\", \"date\": \"Fri, 09 Feb 2024 21:15:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"9472bbea-7078-48ce-aa17-4c8cf275af44\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:44.425714419Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a7880562-4fa3-4578-a14a-7b22f18aed33\", \"content-length\": \"174\", \"date\": \"Fri, 09 Feb 2024 21:15:45 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"a7880562-4fa3-4578-a14a-7b22f18aed33\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:45.385423985Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.117516123Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.117539444Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.117553634Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.117558394Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.117562624Z","hostname":"sock","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.693247111Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57994"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695179277Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695223998Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695250168Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695258648Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695266459Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695273939Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695280899Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.695299229Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"926d42da-b645-44fa-8f5b-82348b9e67a4","remote_addr":"[fd00:1122:3344:101::a]:57994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":2007,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f4ea9957-dedf-4009-86a8-62348c0fd82d\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:46 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"f4ea9957-dedf-4009-86a8-62348c0fd82d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:46.791370343Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.066730757Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55793"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.06850864Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"4e1b043a-5a66-40ef-a9a8-6be02bebb42d","remote_addr":"[fd00:1122:3344:101::a]:55793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1744,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.068750715Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.068778685Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.068974528Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.069697712Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"03669c5c-c100-4efb-aa8f-aefedc7f01ea","remote_addr":"[fd00:1122:3344:101::a]:55793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":997,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.559999221Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40486"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560175945Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560209395Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560247126Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560256506Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560263216Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560270346Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560276626Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.560289967Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"acc0aa5c-9844-47ca-a1c0-f3187c00d978","remote_addr":"[fd00:1122:3344:101::a]:40486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":243,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30ccf0ad-5ee6-4b04-b94b-0b859d942376\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:47 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"30ccf0ad-5ee6-4b04-b94b-0b859d942376\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.619777389Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.712106864Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.712157365Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"06261d69-5001-492b-b81b-08fa17ba51fb","remote_addr":"[fd00:1122:3344:101::a]:34787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.712338568Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.712357828Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.712528111Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:47.712756456Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1908e0e9-be09-4295-b4a2-fd6cafc41726","remote_addr":"[fd00:1122:3344:101::a]:34787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":444,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.960734784Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62859"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.961557599Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.961594149Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.96162774Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.96163513Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.96164084Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.96165103Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.96165937Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:48.961679701Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b97506ad-22b4-4e85-b2d6-6f773a0a4c01","remote_addr":"[fd00:1122:3344:101::a]:62859","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":855,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"db7bebe9-a4fa-4805-8044-cb94b7fa5695\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:49 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"db7bebe9-a4fa-4805-8044-cb94b7fa5695\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:49.043590594Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:49.147621224Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:49.147692125Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"0802078b-ea29-4bcd-8367-e32f5ce70aa4","remote_addr":"[fd00:1122:3344:101::a]:50009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:49.147889679Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:49.14791596Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:49.148167074Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:49.149707102Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"db537aec-4d3b-4e71-a722-573c14163173","remote_addr":"[fd00:1122:3344:101::a]:50009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1846,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.03866534Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34241"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038797533Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038832853Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038863424Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038870864Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038876584Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038882644Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038889094Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.038899614Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6c41f4bf-a7c9-48ce-8c94-dca9d98452be","remote_addr":"[fd00:1122:3344:101::a]:34241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":218,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36333d3f-35c0-44e5-9416-917a93e75beb\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:49 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"36333d3f-35c0-44e5-9416-917a93e75beb\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.108073284Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.205251898Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.205315909Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"58a634fa-89b1-4383-8d58-e3efd46115ab","remote_addr":"[fd00:1122:3344:101::a]:63865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.205478672Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.205511363Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.205731327Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:50.205999602Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"0a257401-94b9-485c-948e-17a9f339266c","remote_addr":"[fd00:1122:3344:101::a]:63865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":549,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.819364567Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42445"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.819508619Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.8195418Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.81957555Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.819584391Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.819590901Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.819597461Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.819604901Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.819631391Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9b85c022-288a-46ad-b243-b82d5612a3ba","remote_addr":"[fd00:1122:3344:101::a]:42445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":245,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c8fd446-f4c2-4cb4-84ea-60fff39d768d\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:51 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2c8fd446-f4c2-4cb4-84ea-60fff39d768d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.887311294Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.995262646Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42503"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.995317436Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"e86df513-047e-4fcb-a429-da62d6133b4e","remote_addr":"[fd00:1122:3344:101::a]:42503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.99547795Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.99549455Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.995691994Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:51.995887917Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"ed257065-c152-4ba7-b99c-b0dd9c82b2b1","remote_addr":"[fd00:1122:3344:101::a]:42503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":443,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315355597Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55423"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.31552933Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315576241Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315589812Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315597152Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315602802Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315608312Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315613462Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.315631672Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e80bd8aa-ffae-4f6b-bee3-fa76d04b1356","remote_addr":"[fd00:1122:3344:101::a]:55423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":230,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d1f65d12-4881-4f45-8bda-87ea324f58b9\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:52 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"d1f65d12-4881-4f45-8bda-87ea324f58b9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.375608233Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.467979679Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47552"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.46805058Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"ed4758f1-d389-47da-8c5d-59f9eb6e6695","remote_addr":"[fd00:1122:3344:101::a]:47552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.468217503Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.468243974Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.468414577Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:53.468606Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"15c09fe8-9207-48d2-b3d3-088286e40a17","remote_addr":"[fd00:1122:3344:101::a]:47552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":407,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.460973447Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60163"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461237021Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461287432Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461319073Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461332603Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461344543Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461356614Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461377264Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.461405914Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b30eb089-772e-43a4-858f-f6f97cf767a1","remote_addr":"[fd00:1122:3344:101::a]:60163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":359,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a4ac5bd-2af2-437a-bf56-587025b692be\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:54 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5a4ac5bd-2af2-437a-bf56-587025b692be\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.52222383Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.632699318Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:35802"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.63335203Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"e688fa11-f06a-4f4d-b289-962787713cfe","remote_addr":"[fd00:1122:3344:101::a]:35802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":632,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.633489322Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.633508653Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.633706226Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:54.63390272Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"8f359939-560d-440f-8508-ae4e2bd065c2","remote_addr":"[fd00:1122:3344:101::a]:35802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":434,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712763974Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63505"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712895026Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712929637Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712957247Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712964557Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712970688Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712988168Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.712998288Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.713003748Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"5910c041-c33f-43e3-8b14-697255f571e5","remote_addr":"[fd00:1122:3344:101::a]:63505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":215,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4c5836e5-1eb2-4899-96c6-24b6bc33c811\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:55 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"4c5836e5-1eb2-4899-96c6-24b6bc33c811\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.768060989Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.861771379Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.86182027Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"19c26644-725d-45c8-b940-5c9fd1a5f9fe","remote_addr":"[fd00:1122:3344:101::a]:33499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.862294619Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.862309629Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.862482932Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:55.862672216Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"2b8bdc35-55aa-4959-9aa5-0e2553d08c38","remote_addr":"[fd00:1122:3344:101::a]:33499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":729,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.247759631Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44833"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.247965644Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.248025746Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.248049586Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.248063786Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.248073896Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.248093197Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.248104067Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.248119577Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6ac49b7c-a5c7-4e28-a3be-4f75390590ca","remote_addr":"[fd00:1122:3344:101::a]:44833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":334,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a39da36a-3f99-4d55-aa92-08d9e4907b2b\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:57 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a39da36a-3f99-4d55-aa92-08d9e4907b2b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.303996713Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.401060295Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.401141746Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"0609e71d-e196-4663-901c-1b9f289c183c","remote_addr":"[fd00:1122:3344:101::a]:43718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.40134839Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.401382341Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.401650186Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:57.40187694Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"8ed96cce-bba2-42eb-9b6f-3163d9cc8adf","remote_addr":"[fd00:1122:3344:101::a]:43718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":562,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.872222615Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36352"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.872413099Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.872455469Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.87247599Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.87248802Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.87251049Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.87252117Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.872531971Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.872541031Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"57bd8b1c-1423-4502-b1e3-efe89217da30","remote_addr":"[fd00:1122:3344:101::a]:36352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":222,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f9c62ff-e3c0-4efe-8daf-223daa78ece0\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:15:59 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"0f9c62ff-e3c0-4efe-8daf-223daa78ece0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:15:59.930044157Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:00.058533565Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:00.058598796Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"b8877db5-b28a-4c7c-a16f-110cf6ae276c","remote_addr":"[fd00:1122:3344:101::a]:53581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:00.05932985Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:00.05935038Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:00.059540774Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:00.059726327Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"79653009-b6ac-40a0-abeb-5524a4553bf1","remote_addr":"[fd00:1122:3344:101::a]:53581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":981,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.476765034Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47211"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.477001618Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.477062139Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.47708683Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.47710614Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.47711316Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.47712116Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.47713152Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.477176401Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c7e24b93-568f-42ce-b2ad-a3ed4a69c9c6","remote_addr":"[fd00:1122:3344:101::a]:47211","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":360,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"325691a8-4622-4eb8-9e59-9f9d1017bab2\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:16:01 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"325691a8-4622-4eb8-9e59-9f9d1017bab2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.555647602Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.666837013Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38696"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.666904224Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"f8bf24e6-4c0f-4f0d-81d2-39b6de0edc1a","remote_addr":"[fd00:1122:3344:101::a]:38696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.667053467Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.667075097Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.667272121Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:02.667474065Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"ba164709-e16f-4262-86b5-b661487775e1","remote_addr":"[fd00:1122:3344:101::a]:38696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":443,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765160788Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41367"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765292611Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765333001Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765350702Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765357502Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765362932Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765369042Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765374552Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.765399752Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"91355b98-4fe8-4b2f-a348-28bf1c8323c8","remote_addr":"[fd00:1122:3344:101::a]:41367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":214,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1dc3e25-eb35-47a4-8ddc-e02c5b388932\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:16:06 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"e1dc3e25-eb35-47a4-8ddc-e02c5b388932\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.817070471Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.910874583Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.911528405Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"dcbf8e41-8b3b-45c9-8e83-3968d2d4dfe3","remote_addr":"[fd00:1122:3344:101::a]:57111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":636,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.911673688Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.911695578Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.911864391Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:07.912053845Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"86a0d33d-332b-4586-a7b9-4a6ee57b1064","remote_addr":"[fd00:1122:3344:101::a]:57111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":399,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.432119026Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54450"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.432278689Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.432311839Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.43234346Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.4323513Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.43235983Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.43237177Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.4323798Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.432398551Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f99591ab-a2aa-4174-b503-1f88dffdf194","remote_addr":"[fd00:1122:3344:101::a]:54450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":242,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c9469238-ad88-474e-9cf2-77ce8718e2d9\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:16:11 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"c9469238-ad88-474e-9cf2-77ce8718e2d9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.488702584Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.585639813Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46951"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.585684944Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"c342d81a-4b20-4996-bc17-f8a09feed049","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46951","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.612034528Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38892"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.612086628Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"d9a628a5-fa1f-4f2d-86d6-2ed53a396d89","remote_addr":"[fd00:1122:3344:101::a]:38892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":35,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.613075236Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.613111597Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.613309001Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:12.613514955Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1282e636-a3d4-403c-86d9-0a97ae46dff6","remote_addr":"[fd00:1122:3344:101::a]:38892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1238,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164737859Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53667"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164884922Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164932943Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164949883Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164956383Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164963114Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164970604Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.164977794Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.165003624Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"33285dfc-14f9-4724-9e3f-a4e8abeb6772","remote_addr":"[fd00:1122:3344:101::a]:53667","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":241,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b10d6bb3-33cb-4603-9ea7-b1325acb0f4b\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:16:18 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b10d6bb3-33cb-4603-9ea7-b1325acb0f4b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.222324136Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.337100173Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.337157024Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"e2f47554-845e-44e1-be6d-12128a01a8eb","remote_addr":"[fd00:1122:3344:101::a]:33935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.337798916Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.337825727Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.33801577Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:19.338234354Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"756127cc-79ec-47d6-8ebb-7af649593504","remote_addr":"[fd00:1122:3344:101::a]:33935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":959,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:21.213866793Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:46521"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:21.218936906Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"c3e0b392-1586-4804-b128-617fa79cd19d","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":5048,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.179755795Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37269"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.179915998Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.179963259Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.17998461Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.17999155Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.17999881Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.18000615Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.18002194Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.180035671Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e613ec81-fd7f-47f7-b398-1b16da4f06b8","remote_addr":"[fd00:1122:3344:101::a]:37269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":251,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6cc30788-73cf-42c5-87d7-a9c14c057587\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:16:28 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"6cc30788-73cf-42c5-87d7-a9c14c057587\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.264950809Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.36958317Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.369860375Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"85fa2796-9e9a-4393-a884-9a9c34d7165f","remote_addr":"[fd00:1122:3344:101::a]:52622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":252,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.371091288Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.371111638Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.371339482Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:29.371508975Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"576059a5-e629-438a-bffa-6d90eb5e9b8e","remote_addr":"[fd00:1122:3344:101::a]:52622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":436,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.67976058Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40520"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.679950163Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.679988264Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.680008974Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.680020854Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.680030424Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.680057055Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.680066905Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.680076695Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2625d25c-f28c-44fe-8f51-963bb54d83a8","remote_addr":"[fd00:1122:3344:101::a]:40520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":243,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a68f238-1154-4755-b71c-d88fd68e2e61\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:16:35 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"6a68f238-1154-4755-b71c-d88fd68e2e61\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.746947563Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.842039608Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54990"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.842094829Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"40cdb987-a0e5-42af-97d6-26913d924c8a","remote_addr":"[fd00:1122:3344:101::a]:54990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":35,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.842225622Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.842257992Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.842442726Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:35.842619679Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"a6b219f0-b4e3-4a29-987e-ddc4ab8d09f2","remote_addr":"[fd00:1122:3344:101::a]:54990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":412,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:42.620006024Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:42.620053265Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a91df899-5f9e-4025-b34f-e9aa4c6defdb","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33582","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154273664Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34855"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154426257Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154458798Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154477718Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154485598Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154494208Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154501859Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154508559Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.154520959Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fd1eb80d-5db9-4953-9259-e511305fbf9f","remote_addr":"[fd00:1122:3344:101::a]:34855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":220,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"39b8067e-9b6a-46da-9030-592e370f259f\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:16:50 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"39b8067e-9b6a-46da-9030-592e370f259f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.212617195Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.330319716Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53168"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.330396717Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"b0bcaf66-3101-4d05-b6b1-0dc177d4d604","remote_addr":"[fd00:1122:3344:101::a]:53168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.330647782Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.330668622Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.330925227Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:50.331109401Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"6d13d0d2-d6df-4f7d-954e-16dc5652233d","remote_addr":"[fd00:1122:3344:101::a]:53168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":483,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:16:51.214294592Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"377684ed-6025-4d74-844f-1a00d41fd1fd","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":201,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:03.002988796Z","hostname":"sock","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542375924Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61460"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542508337Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542554687Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542565238Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542572288Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542578608Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542585328Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542593078Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.542607288Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3aca51aa-db64-4900-a27c-4f38f7fb53fa","remote_addr":"[fd00:1122:3344:101::a]:61460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":216,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec317e7f-5511-4c1b-8be9-ec7674a2bc41\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:17:06 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ec317e7f-5511-4c1b-8be9-ec7674a2bc41\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.591253521Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.68322317Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.683285821Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"e14d731f-7c8c-496d-80d1-1e7bfd81c32d","remote_addr":"[fd00:1122:3344:101::a]:48710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.683434864Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.683453824Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.683659168Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:06.683820171Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"b453f046-4208-49b2-95a3-f7a765a4faf7","remote_addr":"[fd00:1122:3344:101::a]:48710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":410,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:12.653562101Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36747"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:12.653605622Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"b31e4a01-bbcc-4035-9e60-02c4d3377de8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36747","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.214004428Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"4dea4799-91f2-4714-b025-3a0147390dfb","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":208,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.658816332Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46438"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659013866Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659085947Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659109207Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659120758Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659130368Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659146918Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659156988Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.659181319Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8c35344d-baee-4197-955f-87848ca32e4e","remote_addr":"[fd00:1122:3344:101::a]:46438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":320,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a4ae5488-957b-45a3-a629-5f2b2c6d05fa\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:17:21 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a4ae5488-957b-45a3-a629-5f2b2c6d05fa\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.748768573Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.935175075Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55427"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.935237786Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"c5f02d2b-9509-472e-a522-77b410d77162","remote_addr":"[fd00:1122:3344:101::a]:55427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.936104092Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.936122663Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.936365597Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:21.93653972Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"17094517-5778-4a37-8639-531cfc7a0442","remote_addr":"[fd00:1122:3344:101::a]:55427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177466843Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39470"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177641757Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177692297Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177710288Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177720228Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177747628Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177754848Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177761429Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.177768309Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"91c64862-ed72-4297-8e1b-152f6550e938","remote_addr":"[fd00:1122:3344:101::a]:39470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":256,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"46f2880d-2069-4b4f-8404-f6f7cf8e574c\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:17:31 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"46f2880d-2069-4b4f-8404-f6f7cf8e574c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.240655053Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.355618573Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57038"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.355699955Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"6728e51b-877b-4f67-8053-433053a6568f","remote_addr":"[fd00:1122:3344:101::a]:57038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.355870338Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.355890938Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.356138753Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:32.356321037Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"4029d2aa-81d5-4f13-9a10-cb910b60c92d","remote_addr":"[fd00:1122:3344:101::a]:57038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":471,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:42.686537608Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40141"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:42.687074668Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"86d3c1d9-ece1-4aea-b39c-d400d52c4aa2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40141","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":501,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023241871Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44272"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023466545Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023518286Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023547866Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023559846Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023569597Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023581277Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023593567Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.023614347Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"58f193e4-ed2d-464f-9c8c-9089cd4c592f","remote_addr":"[fd00:1122:3344:101::a]:44272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":332,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f1f7a40-12ef-4683-a8b2-6d44cf572af5\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:17:45 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"3f1f7a40-12ef-4683-a8b2-6d44cf572af5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.077560838Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.169328752Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38070"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.169384703Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"a40366d1-5d93-4bcb-982f-7b66776e19bc","remote_addr":"[fd00:1122:3344:101::a]:38070","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.17029723Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.17031735Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.170572245Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:45.170749538Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"cef6ec52-2979-4a97-bd43-570bacfa12d7","remote_addr":"[fd00:1122:3344:101::a]:38070","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1213,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:17:51.214150491Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"f859fb5e-601a-4792-92e7-7600d783a6e7","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712689959Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64431"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712864172Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712911343Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712924463Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712935713Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712945584Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712955674Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712966994Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.712977084Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"52af2d85-b39d-42e0-8235-11417e15bb7e","remote_addr":"[fd00:1122:3344:101::a]:64431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":238,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.719914392Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.719949062Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"f98c37ec-1374-4b87-b230-580354cb0e5f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36617","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":21,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73474ddf-0307-4040-aba1-83d28b3b798b\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:18:12 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"73474ddf-0307-4040-aba1-83d28b3b798b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.770972829Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.875465737Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57905"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.875537508Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"bc70ae7d-5594-4bb5-88a2-4e5aa60ef866","remote_addr":"[fd00:1122:3344:101::a]:57905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.875728202Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.875760182Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.876058698Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:12.876334483Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"47afcb3c-bc9b-4387-98cf-1939a37bed58","remote_addr":"[fd00:1122:3344:101::a]:57905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":636,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:21.213600412Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"f5d35fc6-4c12-4a57-bc52-ae2e4f6972c9","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":287,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.351938072Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57397"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352095945Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352137546Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352227667Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352239357Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352248308Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352274988Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352284648Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.352295009Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f9c19ec2-6f6d-474b-9721-15e847921c37","remote_addr":"[fd00:1122:3344:101::a]:57397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":253,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a3a11bc5-7a2e-40c0-a2cf-5aaedab02e2b\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:18:36 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a3a11bc5-7a2e-40c0-a2cf-5aaedab02e2b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.408920298Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.501653169Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.501723951Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"21cd1424-8a82-43dc-98b1-a1db8921641a","remote_addr":"[fd00:1122:3344:101::a]:58603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.501865663Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.501896564Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.502117968Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:36.502277171Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"96b630f6-04d5-4be2-9f75-f500cf7d2300","remote_addr":"[fd00:1122:3344:101::a]:58603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":435,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:42.753401847Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58789"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:42.753443637Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"f18640f5-39a6-4edc-aa3f-6dad0ebc5d2e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58789","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:18:51.214147622Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"f01ca13b-ad0e-45b7-8a86-06e046c55c41","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":299,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.380776665Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33159"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.380924667Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.380979248Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.380992689Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.381009249Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.381018749Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.38103002Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.38104099Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.381051Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d2dc4643-eda7-471d-9ab0-52db999feb11","remote_addr":"[fd00:1122:3344:101::a]:33159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":256,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7e7505f8-8799-4771-aa6c-e8cb466ecab1\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:19:00 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"7e7505f8-8799-4771-aa6c-e8cb466ecab1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.439502993Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.54885728Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65353"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.548929761Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"8a227bea-66b5-405a-9315-d41069f16070","remote_addr":"[fd00:1122:3344:101::a]:65353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.549124214Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.549148755Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.549502192Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:00.549750786Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"f0a8dd8b-002d-4061-9955-74f53498b39c","remote_addr":"[fd00:1122:3344:101::a]:65353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":668,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:12.78576402Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43957"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:12.786861001Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"1022a3a7-1808-4117-b3bd-47868d2c56e7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43957","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1062,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:21.214093741Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"1dc0db63-2a10-45bd-84db-8080b2be2a94","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.939438127Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38376"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.93958949Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.93963459Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.939655801Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.939663331Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.939682942Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.939690582Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.939701682Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.939718832Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"414a0ee9-06ae-4ddb-803d-c704146b2f37","remote_addr":"[fd00:1122:3344:101::a]:38376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":260,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6e37c1ba-5979-4008-b695-ea104f82484e\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:19:28 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"6e37c1ba-5979-4008-b695-ea104f82484e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:28.995758371Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:29.104988666Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:32944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:29.105708489Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"722eb6fe-239c-482d-b5c4-881ab3ccb516","remote_addr":"[fd00:1122:3344:101::a]:32944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":695,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:29.105863982Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:29.105889442Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:29.106100206Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:29.106269529Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"9d7ae5a2-47f5-4476-8e4a-68d8e4d523c1","remote_addr":"[fd00:1122:3344:101::a]:32944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":424,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:42.8211445Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34990"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:42.821217561Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"648f78ec-fde8-476c-a6fe-4d49bfe9e84f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34990","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:51.21347118Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"97ad0b2e-2413-4940-95f2-42094134ab64","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":214,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.282367062Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41508"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.282615597Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.282679148Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.282712039Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.282719089Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.282733869Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.28274522Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.28277247Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.282810701Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"200f82cf-e7d4-4a4b-8411-823c1436bbc9","remote_addr":"[fd00:1122:3344:101::a]:41508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":362,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"abd94791-04e4-4abf-b3af-f4663e7614ae\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:19:53 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"abd94791-04e4-4abf-b3af-f4663e7614ae\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.339042273Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.454573284Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:35899"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.454634695Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"4aa00512-9cec-4be7-a72c-cbf8ca65251e","remote_addr":"[fd00:1122:3344:101::a]:35899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.454769787Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.454800618Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.455037362Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:19:54.455229406Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"b8a0e485-b223-4641-9948-34e0328466d8","remote_addr":"[fd00:1122:3344:101::a]:35899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":482,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:12.853196958Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:12.8532797Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"500d0964-cec8-4f61-8849-d013deeeb892","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52972","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:21.213780016Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"f7a559d8-f51e-4e9c-8ef5-c0648fcf5cce","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":197,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:42.899353186Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57166"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:42.899393907Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"7b02877a-b044-4380-86f2-9e05d620468f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57166","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.572669698Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54546"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573419791Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573463642Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573485333Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573494333Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573502873Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573509873Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573526943Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.573538174Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"033db7ec-5962-4cbb-a983-14089a5bfcb9","remote_addr":"[fd00:1122:3344:101::a]:54546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":850,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fa4a10f3-4463-4ffd-a912-794e01d0de55\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:20:50 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"fa4a10f3-4463-4ffd-a912-794e01d0de55\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.668476496Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.774387461Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36677"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.774474832Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"bac827ae-625c-4f45-9ab3-1ec7b50be0f4","remote_addr":"[fd00:1122:3344:101::a]:36677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":55,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.775128724Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.775155704Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.775390459Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:50.775569042Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"c506ecd3-bd9c-4022-bb2d-1cc8ab4ffcad","remote_addr":"[fd00:1122:3344:101::a]:36677","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":908,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:20:51.214214994Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"28d3a24b-5c31-4d6a-81f9-6cdaae191a52","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":933,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:12.932630107Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:12.933345631Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"d603ca8e-7a8a-416f-98c5-b538db2e8a9f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56634","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":690,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:14.827596782Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:14.828451587Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:14.871062139Z","hostname":"sock","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.99999786s","next_cleanup":"Instant { tv_sec: 3616, tv_nsec: 193791036 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:15.980116487Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50899"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:15.986459294Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:228","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-09T21:21:15.670880Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(4808884))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-43de7508868\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: 2b2cf3a9-80f2-4ae9-974d-bff21565ea0a, read_only_parent: Some(Volume { block_size: 512, id: 2aa572d6-8d21-42e4-9a36-c64669a332d1, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e21f8714-adaa-4cf9-87b4-660867839361, key: Some(\"9FBVSNBgYSTOl0dM3Z3m5ZP/swQiLy54twrUwlYI91s=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2b2cf3a9-80f2-4ae9-974d-bff21565ea0a, key: Some(\"rqukQwSz2X39YhiNjbikumptFXDuZRBf3h9fnZe9wN4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADCp6pJWElYAACnqklYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAwqeqSVhJWAAAp6pJWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3ZDk4NWE0ZC04ZTNhLTQxZmMtOTM3ZS0xZDgwOGJjZWJmMTEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQzI3T0RkUmkrV0hBT1BFNTUwL2NDRWh4aE12MXBDRjNtajRUakpINWpsRiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:15.986588146Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:262","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:15.987958261Z","hostname":"sock","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance.rs:706","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(4808884))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 45815799-99ad-46db-8a82-30de2f80d1d1, kind: Instance { id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 100, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(4808884), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-43de7508868\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: 2b2cf3a9-80f2-4ae9-974d-bff21565ea0a, read_only_parent: Some(Volume { block_size: 512, id: 2aa572d6-8d21-42e4-9a36-c64669a332d1, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e21f8714-adaa-4cf9-87b4-660867839361, key: Some(\"9FBVSNBgYSTOl0dM3Z3m5ZP/swQiLy54twrUwlYI91s=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2b2cf3a9-80f2-4ae9-974d-bff21565ea0a, key: Some(\"rqukQwSz2X39YhiNjbikumptFXDuZRBf3h9fnZe9wN4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADCp6pJWElYAACnqklYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAwqeqSVhJWAAAp6pJWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3ZDk4NWE0ZC04ZTNhLTQxZmMtOTM3ZS0xZDgwOGJjZWJmMTEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQzI3T0RkUmkrV0hBT1BFNTUwL2NDRWh4aE12MXBDRjNtajRUakpINWpsRiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-09T21:21:15.670880Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:15.988030563Z","hostname":"sock","pid":1671,"uri":"/instances/7d985a4d-8e3a-41fc-937e-1d808bcebf11","method":"PUT","req_id":"5dc15588-5d1b-46f1-a3c0-c68a6c5b037b","remote_addr":"[fd00:1122:3344:101::a]:50899","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7789,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:16.085098454Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53347"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:16.106809783Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", _ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 251, 100, 5]), slot: 0, vni: Vni { inner: 4808884 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:16.107492065Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:16.138332482Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:16.157903041Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:17.932896732Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:17.933071506Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 65.418608ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:18.107981237Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 75.418494ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:18.210520419Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 182.614187ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:18.301724713Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 377.222475ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:18.495330146Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 943.444552ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:18.88289233Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 1.42331653s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:19.838669525Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:21.214112953Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"704848d5-7a5f-42b7-9ccf-ff9b31c8eff7","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":600,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 1.327532625s","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:21.272654357Z","hostname":"sock","pid":1671,"zone":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:22.628122388Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:1098"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 45.512642ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:22.63699288Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 91.854584ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:22.693263213Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 187.797484ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:22.795051932Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\") failed: Property not found. retry in 494.009139ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:22.993892862Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.498532565Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:1107"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.534813421Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:1121"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADCp6pJWElYAACnqklYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAwqeqSVhJWAAAp6pJWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3ZDk4NWE0ZC04ZTNhLTQxZmMtOTM3ZS0xZDgwOGJjZWJmMTEiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQzI3T0RkUmkrV0hBT1BFNTUwL2NDRWh4aE12MXBDRjNtajRUakpINWpsRiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-43de7508868\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: 2b2cf3a9-80f2-4ae9-974d-bff21565ea0a, read_only_parent: Some(Volume { block_size: 512, id: 2aa572d6-8d21-42e4-9a36-c64669a332d1, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e21f8714-adaa-4cf9-87b4-660867839361, key: Some(\"9FBVSNBgYSTOl0dM3Z3m5ZP/swQiLy54twrUwlYI91s=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2b2cf3a9-80f2-4ae9-974d-bff21565ea0a, key: Some(\"rqukQwSz2X39YhiNjbikumptFXDuZRBf3h9fnZe9wN4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 7d985a4d-8e3a-41fc-937e-1d808bcebf11, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.536034703Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:453"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.685699221Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:455"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.686005316Z","hostname":"sock","pid":1671,"uri":"/instances/7d985a4d-8e3a-41fc-937e-1d808bcebf11/state","method":"PUT","req_id":"11898048-9314-4f0b-9367-9d8ddf9dda92","remote_addr":"[fd00:1122:3344:101::a]:53347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7600810,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-02-09T21:21:23.686238040Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.686256131Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.686273271Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-09T21:21:23.686238040Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }","propolis_id":"0d5e0888-ccca-4987-9a7b-348747d4526e"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.686296942Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }, propolis_id: 0d5e0888-ccca-4987-9a7b-348747d4526e, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-09T21:21:23.686238040Z } }","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-02-09T21:21:23.723839741Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.723860491Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.723878991Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-09T21:21:23.723839741Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }","propolis_id":"0d5e0888-ccca-4987-9a7b-348747d4526e"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:23.723889062Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }, propolis_id: 0d5e0888-ccca-4987-9a7b-348747d4526e, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-09T21:21:23.723839741Z } }","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.533787898Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55588"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.533953271Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 141301e9-b85a-47cc-a0f7-f99fb76ca99c, kind: Service { id: 2130b665-8c04-446a-ae31-3ccbd36963c1 }, name: Name(\"external-dns-2130b665-8c04-446a-ae31-3ccbd36963c1\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 134, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: abff41cf-6397-4efe-9c5a-3207cfe197e7, kind: Service { id: 26fa6230-6a52-4867-a073-2918d92baa6b }, name: Name(\"external-dns-26fa6230-6a52-4867-a073-2918d92baa6b\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 219, 17])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06e3aaef-9a37-4c02-882e-baa48e65f7a5, kind: Service { id: b9446a8d-d093-495b-8393-dd3553e581a3 }, name: Name(\"nexus-b9446a8d-d093-495b-8393-dd3553e581a3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 252710e7-00ff-418e-9422-610fdf075322, kind: Service { id: ba0d150e-a49b-428d-a98d-600e9c760fb3 }, name: Name(\"nexus-ba0d150e-a49b-428d-a98d-600e9c760fb3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 144, 141])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 93aa5154-032c-40b9-a5f9-22d84c0046a8, kind: Service { id: 5505a386-30f2-436c-a5d3-2bc80bc9d560 }, name: Name(\"nexus-5505a386-30f2-436c-a5d3-2bc80bc9d560\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.533989172Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.534030663Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.534042823Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.534049723Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.534058253Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.534065573Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.534084934Z","hostname":"sock","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e148ad5e-6fdc-4520-a8f9-6ee5296e0e8b","remote_addr":"[fd00:1122:3344:101::a]:55588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":264,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73ee1dde-813c-4e55-8e66-8bb1ffb42508\", \"content-length\": \"127\", \"date\": \"Fri, 09 Feb 2024 21:21:39 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"73ee1dde-813c-4e55-8e66-8bb1ffb42508\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.633427507Z","hostname":"sock","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.879870741Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37929"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.879961973Z","hostname":"sock","pid":1671,"uri":"/inventory","method":"GET","req_id":"c7342dd0-87c7-4190-bbee-734abd5588d1","remote_addr":"[fd00:1122:3344:101::a]:37929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.880178207Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.880219117Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.880580434Z","hostname":"sock","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:39.88087354Z","hostname":"sock","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"84ff6c3b-b883-489a-9c4a-c593f7e4ea0f","remote_addr":"[fd00:1122:3344:101::a]:37929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":737,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:42.965749906Z","hostname":"sock","pid":1671,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49446"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:42.966453809Z","hostname":"sock","pid":1671,"uri":"/baseboard","method":"GET","req_id":"91fe710d-91e3-406a-9721-f7bef30e7524","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49446","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":674,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.889527472Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44756"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.891286855Z","hostname":"sock","pid":1671,"uri":"/instances/7d985a4d-8e3a-41fc-937e-1d808bcebf11/state","method":"PUT","req_id":"af090ec2-84af-4653-ad67-874c55d7dafd","remote_addr":"[fd00:1122:3344:101::a]:44756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1740,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-02-09T21:21:48.891332776Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.891347016Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.891356986Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-09T21:21:48.891332776Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }","propolis_id":"0d5e0888-ccca-4987-9a7b-348747d4526e"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.891367136Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }, propolis_id: 0d5e0888-ccca-4987-9a7b-348747d4526e, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-09T21:21:48.891332776Z } }","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopped), migration_status: NoMigration, time: 2024-02-09T21:21:48.950707295Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.950723396Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.950736346Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-09T21:21:48.950707295Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }","propolis_id":"0d5e0888-ccca-4987-9a7b-348747d4526e"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:48.950746016Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(0d5e0888-ccca-4987-9a7b-348747d4526e), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-09T21:21:15.696254567Z }, propolis_id: 0d5e0888-ccca-4987-9a7b-348747d4526e, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-09T21:21:48.950707295Z } }","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-02-09T21:21:49.077912571Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:49.077951781Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:49.077980592Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-09T21:21:49.077912571Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-09T21:21:49.077912571Z }","propolis_id":"0d5e0888-ccca-4987-9a7b-348747d4526e"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:49.077991912Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:374","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:49.077997572Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:49.078156575Z","hostname":"sock","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:49.078508521Z","hostname":"sock","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:49.931856296Z","hostname":"sock","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1229","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e\", bundle_id: 0a8fd597-6520-4f3b-90f7-e072ec32edbd }, time_created: 2024-02-09T21:21:49.078712325Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_0d5e0888-ccca-4987-9a7b-348747d4526e","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:49.931938687Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:51.213873859Z","hostname":"sock","pid":1671,"uri":"/metrics/collect/e8553772-9db2-44b1-acc1-a08a65e40999","method":"GET","req_id":"d247de36-585c-46c5-b774-58b7c02d2645","remote_addr":"[fd00:1122:3344:101::d]:46521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":637,"response_code":"200"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:51.509289951Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:51.534071386Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-09T21:21:49.077912571Z }, propolis_id: 0d5e0888-ccca-4987-9a7b-348747d4526e, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-09T21:21:49.077912571Z } }","instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:51.603761955Z","hostname":"sock","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34622"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-02-09T21:21:51.60459793Z","hostname":"sock","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:51.604620761Z","hostname":"sock","pid":1671,"uri":"/v2p/45815799-99ad-46db-8a82-30de2f80d1d1","method":"DELETE","req_id":"aded2eec-1969-4797-bd7e-ef913b5cf02a","remote_addr":"[fd00:1122:3344:101::a]:34622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":747,"response_code":"204"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-02-09T21:21:51.945023029Z","hostname":"sock","pid":1671,"instance_id":"7d985a4d-8e3a-41fc-937e-1d808bcebf11","component":"InstanceManager","file":"sled-agent/src/instance.rs:490"}